2024-11-09 21:15:28,922 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 21:15:28,938 main DEBUG Took 0.012935 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 21:15:28,938 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 21:15:28,938 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 21:15:28,940 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 21:15:28,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,950 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 21:15:28,962 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,965 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,966 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,966 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,968 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,968 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,969 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,970 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,971 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,972 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,972 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,973 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,973 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,974 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,974 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,975 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,975 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,976 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,976 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,977 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 21:15:28,978 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,978 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 21:15:28,980 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 21:15:28,982 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 21:15:28,985 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 21:15:28,985 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 21:15:28,987 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 21:15:28,987 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 21:15:28,995 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 21:15:28,998 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 21:15:28,999 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 21:15:28,999 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 21:15:29,000 main DEBUG createAppenders(={Console}) 2024-11-09 21:15:29,000 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-09 21:15:29,001 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-09 21:15:29,001 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-09 21:15:29,001 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 21:15:29,002 main DEBUG OutputStream closed 2024-11-09 21:15:29,002 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 21:15:29,002 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 21:15:29,002 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-09 21:15:29,074 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 21:15:29,076 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 21:15:29,077 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 21:15:29,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 21:15:29,078 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 21:15:29,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 21:15:29,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 21:15:29,079 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 21:15:29,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 21:15:29,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 21:15:29,080 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 21:15:29,081 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 21:15:29,081 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 21:15:29,081 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 21:15:29,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 21:15:29,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 21:15:29,082 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 21:15:29,083 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 21:15:29,085 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 21:15:29,086 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-09 21:15:29,086 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 21:15:29,087 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-09T21:15:29,301 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775 2024-11-09 21:15:29,305 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 21:15:29,305 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T21:15:29,315 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-09T21:15:29,338 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T21:15:29,358 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446, deleteOnExit=true 2024-11-09T21:15:29,359 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T21:15:29,360 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/test.cache.data in system properties and HBase conf 2024-11-09T21:15:29,361 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T21:15:29,362 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/hadoop.log.dir in system properties and HBase conf 2024-11-09T21:15:29,363 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T21:15:29,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T21:15:29,364 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T21:15:29,476 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T21:15:29,560 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T21:15:29,564 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T21:15:29,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T21:15:29,565 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T21:15:29,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T21:15:29,566 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T21:15:29,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T21:15:29,567 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T21:15:29,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T21:15:29,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T21:15:29,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/nfs.dump.dir in system properties and HBase conf 2024-11-09T21:15:29,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/java.io.tmpdir in system properties and HBase conf 2024-11-09T21:15:29,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T21:15:29,569 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T21:15:29,570 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T21:15:30,953 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T21:15:31,023 INFO [Time-limited test {}] log.Log(170): Logging initialized @2770ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T21:15:31,090 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:15:31,146 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:15:31,167 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:15:31,167 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:15:31,168 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T21:15:31,179 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:15:31,182 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:15:31,184 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:15:31,358 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2341a969{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/java.io.tmpdir/jetty-localhost-35323-hadoop-hdfs-3_4_1-tests_jar-_-any-9556599601459714800/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T21:15:31,364 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:35323} 2024-11-09T21:15:31,365 INFO [Time-limited test {}] server.Server(415): Started @3113ms 2024-11-09T21:15:31,944 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T21:15:31,951 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T21:15:31,952 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T21:15:31,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T21:15:31,953 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T21:15:31,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/hadoop.log.dir/,AVAILABLE} 2024-11-09T21:15:31,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T21:15:32,053 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@292d50c4{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/java.io.tmpdir/jetty-localhost-39373-hadoop-hdfs-3_4_1-tests_jar-_-any-17178060967649469806/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:15:32,053 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:39373} 2024-11-09T21:15:32,053 INFO [Time-limited test {}] server.Server(415): Started @3802ms 2024-11-09T21:15:32,101 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T21:15:33,099 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/data/data1/current/BP-1250636572-172.17.0.2-1731186930089/current, will proceed with Du for space computation calculation, 2024-11-09T21:15:33,099 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/data/data2/current/BP-1250636572-172.17.0.2-1731186930089/current, will proceed with Du for space computation calculation, 2024-11-09T21:15:33,125 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T21:15:33,167 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x10abb69f2eeb5e29 with lease ID 0xb6991c98f725a269: Processing first storage report for DS-7b6dc24e-7022-484c-b730-9a67a7cd716a from datanode DatanodeRegistration(127.0.0.1:43329, datanodeUuid=f48c03a0-7fd7-4b12-8a65-5a278ca37ae8, infoPort=33709, infoSecurePort=0, ipcPort=39545, storageInfo=lv=-57;cid=testClusterID;nsid=1925563868;c=1731186930089) 2024-11-09T21:15:33,169 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x10abb69f2eeb5e29 with lease ID 0xb6991c98f725a269: from storage DS-7b6dc24e-7022-484c-b730-9a67a7cd716a node DatanodeRegistration(127.0.0.1:43329, datanodeUuid=f48c03a0-7fd7-4b12-8a65-5a278ca37ae8, infoPort=33709, infoSecurePort=0, ipcPort=39545, storageInfo=lv=-57;cid=testClusterID;nsid=1925563868;c=1731186930089), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T21:15:33,169 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x10abb69f2eeb5e29 with lease ID 0xb6991c98f725a269: Processing first storage report for DS-92b75273-d691-4d37-9860-e4fd6ab8e4fc from datanode DatanodeRegistration(127.0.0.1:43329, datanodeUuid=f48c03a0-7fd7-4b12-8a65-5a278ca37ae8, infoPort=33709, infoSecurePort=0, ipcPort=39545, storageInfo=lv=-57;cid=testClusterID;nsid=1925563868;c=1731186930089) 2024-11-09T21:15:33,170 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x10abb69f2eeb5e29 with lease ID 0xb6991c98f725a269: from storage DS-92b75273-d691-4d37-9860-e4fd6ab8e4fc node DatanodeRegistration(127.0.0.1:43329, datanodeUuid=f48c03a0-7fd7-4b12-8a65-5a278ca37ae8, infoPort=33709, infoSecurePort=0, ipcPort=39545, storageInfo=lv=-57;cid=testClusterID;nsid=1925563868;c=1731186930089), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T21:15:33,225 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775 2024-11-09T21:15:33,315 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/zookeeper_0, clientPort=59129, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T21:15:33,325 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59129 2024-11-09T21:15:33,334 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:33,336 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:33,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741825_1001 (size=7) 2024-11-09T21:15:33,945 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 with version=8 2024-11-09T21:15:33,945 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/hbase-staging 2024-11-09T21:15:34,026 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T21:15:34,254 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9514d6d5e5be:0 server-side Connection retries=45 2024-11-09T21:15:34,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:15:34,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:15:34,267 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:15:34,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:15:34,268 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:15:34,400 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T21:15:34,457 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T21:15:34,465 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T21:15:34,469 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:15:34,491 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8407 (auto-detected) 2024-11-09T21:15:34,492 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-09T21:15:34,509 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:34371 2024-11-09T21:15:34,529 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:34371 connecting to ZooKeeper ensemble=127.0.0.1:59129 2024-11-09T21:15:34,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:343710x0, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:15:34,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:34371-0x101217439d90000 connected 2024-11-09T21:15:34,744 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:34,746 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:34,755 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:15:34,758 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982, hbase.cluster.distributed=false 2024-11-09T21:15:34,780 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:15:34,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=34371 2024-11-09T21:15:34,785 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=34371 2024-11-09T21:15:34,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=34371 2024-11-09T21:15:34,786 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=34371 2024-11-09T21:15:34,787 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=34371 2024-11-09T21:15:34,888 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9514d6d5e5be:0 server-side Connection retries=45 2024-11-09T21:15:34,889 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:15:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T21:15:34,890 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T21:15:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T21:15:34,890 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T21:15:34,893 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T21:15:34,896 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T21:15:34,897 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38413 2024-11-09T21:15:34,899 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38413 connecting to ZooKeeper ensemble=127.0.0.1:59129 2024-11-09T21:15:34,900 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:34,902 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:34,919 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:384130x0, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T21:15:34,920 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38413-0x101217439d90001 connected 2024-11-09T21:15:34,920 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:384130x0, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:15:34,924 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T21:15:34,931 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T21:15:34,934 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T21:15:34,939 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T21:15:34,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38413 2024-11-09T21:15:34,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38413 2024-11-09T21:15:34,940 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38413 2024-11-09T21:15:34,941 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38413 2024-11-09T21:15:34,942 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38413 2024-11-09T21:15:34,956 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9514d6d5e5be:34371 2024-11-09T21:15:34,957 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:34,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:15:34,975 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:15:34,977 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:35,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T21:15:35,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:35,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:35,007 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T21:15:35,008 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9514d6d5e5be,34371,1731186934105 from backup master directory 2024-11-09T21:15:35,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:35,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:15:35,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T21:15:35,017 WARN [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:15:35,017 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:35,019 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T21:15:35,020 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T21:15:35,073 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/hbase.id] with ID: 131b0082-dcda-4525-9e4a-2e80d2847bb6 2024-11-09T21:15:35,073 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/.tmp/hbase.id 2024-11-09T21:15:35,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741826_1002 (size=42) 2024-11-09T21:15:35,485 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/.tmp/hbase.id]:[hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/hbase.id] 2024-11-09T21:15:35,540 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T21:15:35,545 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T21:15:35,564 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-09T21:15:35,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:35,631 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:35,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741827_1003 (size=196) 2024-11-09T21:15:36,072 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:15:36,074 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T21:15:36,087 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:36,090 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:15:36,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741828_1004 (size=1189) 2024-11-09T21:15:36,542 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store 2024-11-09T21:15:36,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741829_1005 (size=34) 2024-11-09T21:15:36,965 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T21:15:36,969 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:15:36,970 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T21:15:36,971 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:15:36,971 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:15:36,973 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T21:15:36,973 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:15:36,974 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:15:36,975 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731186936970Disabling compacts and flushes for region at 1731186936970Disabling writes for close at 1731186936973 (+3 ms)Writing region close event to WAL at 1731186936973Closed at 1731186936973 2024-11-09T21:15:36,978 WARN [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/.initializing 2024-11-09T21:15:36,978 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/WALs/9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:36,989 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:15:37,008 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C34371%2C1731186934105, suffix=, logDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/WALs/9514d6d5e5be,34371,1731186934105, archiveDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/oldWALs, maxLogs=10 2024-11-09T21:15:37,036 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/WALs/9514d6d5e5be,34371,1731186934105/9514d6d5e5be%2C34371%2C1731186934105.1731186937014, exclude list is [], retry=0 2024-11-09T21:15:37,052 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43329,DS-7b6dc24e-7022-484c-b730-9a67a7cd716a,DISK] 2024-11-09T21:15:37,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T21:15:37,090 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/WALs/9514d6d5e5be,34371,1731186934105/9514d6d5e5be%2C34371%2C1731186934105.1731186937014 2024-11-09T21:15:37,092 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33709:33709)] 2024-11-09T21:15:37,092 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:15:37,093 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:15:37,097 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,098 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,136 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,160 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T21:15:37,163 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:37,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:37,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T21:15:37,171 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:37,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:15:37,172 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,175 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T21:15:37,176 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:37,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:15:37,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T21:15:37,180 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:37,181 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:15:37,182 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,185 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,186 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,193 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,194 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,198 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T21:15:37,202 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T21:15:37,208 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:15:37,209 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60883127, jitterRate=-0.09277071058750153}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T21:15:37,215 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731186937111Initializing all the Stores at 1731186937114 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186937115 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186937115Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186937116 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186937116Cleaning up temporary data from old regions at 1731186937194 (+78 ms)Region opened successfully at 1731186937215 (+21 ms) 2024-11-09T21:15:37,217 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T21:15:37,245 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d9b5d48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9514d6d5e5be/172.17.0.2:0 2024-11-09T21:15:37,271 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T21:15:37,280 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-09T21:15:37,280 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-09T21:15:37,283 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T21:15:37,284 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T21:15:37,288 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T21:15:37,289 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T21:15:37,311 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T21:15:37,319 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T21:15:37,332 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T21:15:37,334 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T21:15:37,335 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T21:15:37,342 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T21:15:37,345 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T21:15:37,349 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T21:15:37,361 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T21:15:37,364 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T21:15:37,374 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T21:15:37,394 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T21:15:37,405 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T21:15:37,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:15:37,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T21:15:37,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:37,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:37,421 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9514d6d5e5be,34371,1731186934105, sessionid=0x101217439d90000, setting cluster-up flag (Was=false) 2024-11-09T21:15:37,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:37,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:37,479 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T21:15:37,482 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:37,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:37,511 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:37,543 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T21:15:37,548 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:37,557 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T21:15:37,618 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T21:15:37,627 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T21:15:37,632 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T21:15:37,636 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9514d6d5e5be,34371,1731186934105 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T21:15:37,643 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:15:37,643 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:15:37,643 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:15:37,643 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=5, maxPoolSize=5 2024-11-09T21:15:37,643 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9514d6d5e5be:0, corePoolSize=10, maxPoolSize=10 2024-11-09T21:15:37,644 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,644 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:15:37,644 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,645 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731186967645 2024-11-09T21:15:37,646 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T21:15:37,647 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T21:15:37,649 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(746): ClusterId : 131b0082-dcda-4525-9e4a-2e80d2847bb6 2024-11-09T21:15:37,651 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:15:37,651 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T21:15:37,651 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T21:15:37,651 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T21:15:37,652 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T21:15:37,652 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T21:15:37,652 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T21:15:37,653 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,656 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T21:15:37,657 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:37,657 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T21:15:37,658 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T21:15:37,657 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T21:15:37,659 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T21:15:37,660 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T21:15:37,660 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T21:15:37,661 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T21:15:37,661 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.large.0-1731186937661,5,FailOnTimeoutGroup] 2024-11-09T21:15:37,664 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.small.0-1731186937661,5,FailOnTimeoutGroup] 2024-11-09T21:15:37,664 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,665 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T21:15:37,666 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,666 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741831_1007 (size=1321) 2024-11-09T21:15:37,670 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T21:15:37,671 DEBUG [RS:0;9514d6d5e5be:38413 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53d4f39e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9514d6d5e5be/172.17.0.2:0 2024-11-09T21:15:37,683 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9514d6d5e5be:38413 2024-11-09T21:15:37,686 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T21:15:37,686 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T21:15:37,687 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T21:15:37,689 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(2659): reportForDuty to master=9514d6d5e5be,34371,1731186934105 with port=38413, startcode=1731186934857 2024-11-09T21:15:37,699 DEBUG [RS:0;9514d6d5e5be:38413 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T21:15:37,728 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48299, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T21:15:37,733 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34371 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:37,735 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=34371 {}] master.ServerManager(517): Registering regionserver=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:37,748 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:15:37,748 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42817 2024-11-09T21:15:37,748 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T21:15:37,761 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:15:37,762 DEBUG [RS:0;9514d6d5e5be:38413 {}] zookeeper.ZKUtil(111): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:37,762 WARN [RS:0;9514d6d5e5be:38413 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T21:15:37,762 INFO [RS:0;9514d6d5e5be:38413 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:15:37,762 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:37,764 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9514d6d5e5be,38413,1731186934857] 2024-11-09T21:15:37,786 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T21:15:37,801 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T21:15:37,806 INFO [RS:0;9514d6d5e5be:38413 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T21:15:37,806 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,807 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T21:15:37,812 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T21:15:37,813 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,813 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,813 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,813 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,813 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9514d6d5e5be:0, corePoolSize=2, maxPoolSize=2 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,814 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9514d6d5e5be:0, corePoolSize=1, maxPoolSize=1 2024-11-09T21:15:37,815 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:15:37,815 DEBUG [RS:0;9514d6d5e5be:38413 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0, corePoolSize=3, maxPoolSize=3 2024-11-09T21:15:37,816 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,816 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,816 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,816 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,816 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,817 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38413,1731186934857-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:15:37,834 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T21:15:37,836 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,38413,1731186934857-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,836 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,836 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.Replication(171): 9514d6d5e5be,38413,1731186934857 started 2024-11-09T21:15:37,854 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:37,854 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1482): Serving as 9514d6d5e5be,38413,1731186934857, RpcServer on 9514d6d5e5be/172.17.0.2:38413, sessionid=0x101217439d90001 2024-11-09T21:15:37,855 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T21:15:37,855 DEBUG [RS:0;9514d6d5e5be:38413 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:37,855 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,38413,1731186934857' 2024-11-09T21:15:37,855 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T21:15:37,856 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T21:15:37,857 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T21:15:37,857 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T21:15:37,857 DEBUG [RS:0;9514d6d5e5be:38413 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:37,857 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9514d6d5e5be,38413,1731186934857' 2024-11-09T21:15:37,857 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T21:15:37,858 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T21:15:37,859 DEBUG [RS:0;9514d6d5e5be:38413 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T21:15:37,859 INFO [RS:0;9514d6d5e5be:38413 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T21:15:37,859 INFO [RS:0;9514d6d5e5be:38413 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T21:15:37,972 INFO [RS:0;9514d6d5e5be:38413 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T21:15:37,975 INFO [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C38413%2C1731186934857, suffix=, logDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857, archiveDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/oldWALs, maxLogs=32 2024-11-09T21:15:37,989 DEBUG [RS:0;9514d6d5e5be:38413 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857/9514d6d5e5be%2C38413%2C1731186934857.1731186937977, exclude list is [], retry=0 2024-11-09T21:15:37,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43329,DS-7b6dc24e-7022-484c-b730-9a67a7cd716a,DISK] 2024-11-09T21:15:37,997 INFO [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857/9514d6d5e5be%2C38413%2C1731186934857.1731186937977 2024-11-09T21:15:37,997 DEBUG [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33709:33709)] 2024-11-09T21:15:38,078 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T21:15:38,079 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:15:38,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741833_1009 (size=32) 2024-11-09T21:15:38,499 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:15:38,504 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:15:38,508 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:15:38,508 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:38,509 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:38,510 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:15:38,513 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:15:38,514 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:38,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:38,515 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:15:38,518 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:15:38,518 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:38,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:38,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:15:38,522 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:15:38,522 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:38,523 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:38,523 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:15:38,524 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740 2024-11-09T21:15:38,525 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740 2024-11-09T21:15:38,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:15:38,528 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:15:38,529 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:15:38,531 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:15:38,534 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:15:38,535 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67030890, jitterRate=-0.0011619031429290771}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:15:38,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731186938500Initializing all the Stores at 1731186938503 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186938503Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186938503Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186938503Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186938504 (+1 ms)Cleaning up temporary data from old regions at 1731186938528 (+24 ms)Region opened successfully at 1731186938539 (+11 ms) 2024-11-09T21:15:38,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T21:15:38,539 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T21:15:38,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T21:15:38,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T21:15:38,539 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T21:15:38,541 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T21:15:38,541 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731186938539Disabling compacts and flushes for region at 1731186938539Disabling writes for close at 1731186938539Writing region close event to WAL at 1731186938540 (+1 ms)Closed at 1731186938541 (+1 ms) 2024-11-09T21:15:38,545 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:15:38,545 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T21:15:38,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T21:15:38,557 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:15:38,560 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T21:15:38,717 DEBUG [9514d6d5e5be:34371 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-09T21:15:38,730 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:38,736 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,38413,1731186934857, state=OPENING 2024-11-09T21:15:38,795 DEBUG [PEWorker-5 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T21:15:38,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:38,806 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:15:38,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:15:38,808 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:15:38,812 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T21:15:38,817 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:15:38,997 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T21:15:39,001 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38033, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T21:15:39,011 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T21:15:39,012 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T21:15:39,012 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T21:15:39,016 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9514d6d5e5be%2C38413%2C1731186934857.meta, suffix=.meta, logDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857, archiveDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/oldWALs, maxLogs=32 2024-11-09T21:15:39,031 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857/9514d6d5e5be%2C38413%2C1731186934857.meta.1731186939018.meta, exclude list is [], retry=0 2024-11-09T21:15:39,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43329,DS-7b6dc24e-7022-484c-b730-9a67a7cd716a,DISK] 2024-11-09T21:15:39,038 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/WALs/9514d6d5e5be,38413,1731186934857/9514d6d5e5be%2C38413%2C1731186934857.meta.1731186939018.meta 2024-11-09T21:15:39,039 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33709:33709)] 2024-11-09T21:15:39,039 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:15:39,041 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T21:15:39,043 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T21:15:39,048 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T21:15:39,052 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T21:15:39,052 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:15:39,053 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T21:15:39,053 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T21:15:39,056 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T21:15:39,057 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T21:15:39,057 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:39,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:39,058 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T21:15:39,060 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T21:15:39,060 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:39,060 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:39,061 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T21:15:39,062 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T21:15:39,062 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:39,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:39,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T21:15:39,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T21:15:39,065 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:39,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T21:15:39,066 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T21:15:39,067 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740 2024-11-09T21:15:39,070 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740 2024-11-09T21:15:39,073 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T21:15:39,073 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T21:15:39,074 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:15:39,076 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T21:15:39,078 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64596223, jitterRate=-0.03744126856327057}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:15:39,079 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T21:15:39,081 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731186939053Writing region info on filesystem at 1731186939053Initializing all the Stores at 1731186939055 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186939055Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186939055Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186939055Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731186939055Cleaning up temporary data from old regions at 1731186939073 (+18 ms)Running coprocessor post-open hooks at 1731186939079 (+6 ms)Region opened successfully at 1731186939080 (+1 ms) 2024-11-09T21:15:39,087 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731186938987 2024-11-09T21:15:39,097 DEBUG [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T21:15:39,098 INFO [RS_OPEN_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T21:15:39,099 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:39,102 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9514d6d5e5be,38413,1731186934857, state=OPEN 2024-11-09T21:15:39,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:15:39,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T21:15:39,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:15:39,185 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T21:15:39,185 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:39,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T21:15:39,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9514d6d5e5be,38413,1731186934857 in 370 msec 2024-11-09T21:15:39,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T21:15:39,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 643 msec 2024-11-09T21:15:39,201 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T21:15:39,201 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T21:15:39,219 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:39,220 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:39,238 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:39,241 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41111, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:39,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6760 sec 2024-11-09T21:15:39,259 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731186939259, completionTime=-1 2024-11-09T21:15:39,262 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-09T21:15:39,262 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T21:15:39,292 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-09T21:15:39,292 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731186999292 2024-11-09T21:15:39,293 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731187059293 2024-11-09T21:15:39,293 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-09T21:15:39,295 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,34371,1731186934105-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,295 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,34371,1731186934105-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,295 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,34371,1731186934105-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,297 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9514d6d5e5be:34371, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,297 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,297 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,303 DEBUG [master/9514d6d5e5be:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T21:15:39,325 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.307sec 2024-11-09T21:15:39,326 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T21:15:39,327 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T21:15:39,328 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T21:15:39,329 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T21:15:39,329 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T21:15:39,330 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,34371,1731186934105-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T21:15:39,331 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,34371,1731186934105-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T21:15:39,338 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T21:15:39,338 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T21:15:39,339 INFO [master/9514d6d5e5be:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9514d6d5e5be,34371,1731186934105-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T21:15:39,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d57a2b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:39,379 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T21:15:39,379 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T21:15:39,384 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:39,387 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:39,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8ae91c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:39,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:39,408 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55958, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:39,411 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a7fbb0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:39,412 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:39,419 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:39,420 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:39,422 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47692, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:39,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:39,450 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=204, ProcessCount=11, AvailableMemoryMB=3627 2024-11-09T21:15:39,457 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:15:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:39,465 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@49a86ad1 2024-11-09T21:15:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:15:39,469 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:15:39,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:15:39,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:15:39,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T21:15:39,485 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:15:39,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-09T21:15:39,487 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:39,490 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:15:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:15:39,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741835_1011 (size=1061) 2024-11-09T21:15:39,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:15:39,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:15:39,912 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:15:39,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741836_1012 (size=53) 2024-11-09T21:15:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:15:40,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:15:40,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing ac01e9e325a29ffdc19c65c95fe0a1df, disabling compactions & flushes 2024-11-09T21:15:40,326 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,326 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. after waiting 0 ms 2024-11-09T21:15:40,327 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,327 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,327 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for ac01e9e325a29ffdc19c65c95fe0a1df: Waiting for close lock at 1731186940326Disabling compacts and flushes for region at 1731186940326Disabling writes for close at 1731186940327 (+1 ms)Writing region close event to WAL at 1731186940327Closed at 1731186940327 2024-11-09T21:15:40,336 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:15:40,341 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731186940336"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186940336"}]},"ts":"1731186940336"} 2024-11-09T21:15:40,346 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:15:40,348 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:15:40,350 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186940348"}]},"ts":"1731186940348"} 2024-11-09T21:15:40,355 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T21:15:40,356 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, ASSIGN}] 2024-11-09T21:15:40,359 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, ASSIGN 2024-11-09T21:15:40,360 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, ASSIGN; state=OFFLINE, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=false 2024-11-09T21:15:40,513 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ac01e9e325a29ffdc19c65c95fe0a1df, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:40,521 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, ASSIGN because future has completed 2024-11-09T21:15:40,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:15:40,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:15:40,684 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,684 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:15:40,685 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,685 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:15:40,685 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,686 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,689 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,693 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:15:40,694 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac01e9e325a29ffdc19c65c95fe0a1df columnFamilyName A 2024-11-09T21:15:40,694 DEBUG [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:40,695 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.HStore(327): Store=ac01e9e325a29ffdc19c65c95fe0a1df/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:15:40,696 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,698 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:15:40,699 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac01e9e325a29ffdc19c65c95fe0a1df columnFamilyName B 2024-11-09T21:15:40,699 DEBUG [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:40,700 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.HStore(327): Store=ac01e9e325a29ffdc19c65c95fe0a1df/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:15:40,700 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,702 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:15:40,703 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ac01e9e325a29ffdc19c65c95fe0a1df columnFamilyName C 2024-11-09T21:15:40,703 DEBUG [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:15:40,704 INFO [StoreOpener-ac01e9e325a29ffdc19c65c95fe0a1df-1 {}] regionserver.HStore(327): Store=ac01e9e325a29ffdc19c65c95fe0a1df/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:15:40,704 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,704 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,706 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,707 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,709 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,709 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,711 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:15:40,714 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,718 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:15:40,719 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened ac01e9e325a29ffdc19c65c95fe0a1df; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60347173, jitterRate=-0.10075704753398895}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:15:40,719 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:40,720 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for ac01e9e325a29ffdc19c65c95fe0a1df: Running coprocessor pre-open hook at 1731186940686Writing region info on filesystem at 1731186940686Initializing all the Stores at 1731186940688 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186940688Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186940689 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186940689Cleaning up temporary data from old regions at 1731186940710 (+21 ms)Running coprocessor post-open hooks at 1731186940719 (+9 ms)Region opened successfully at 1731186940720 (+1 ms) 2024-11-09T21:15:40,722 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., pid=6, masterSystemTime=1731186940676 2024-11-09T21:15:40,726 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,726 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:40,727 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ac01e9e325a29ffdc19c65c95fe0a1df, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:40,731 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:15:40,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-09T21:15:40,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 in 211 msec 2024-11-09T21:15:40,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-09T21:15:40,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, ASSIGN in 382 msec 2024-11-09T21:15:40,744 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:15:40,745 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186940744"}]},"ts":"1731186940744"} 2024-11-09T21:15:40,748 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T21:15:40,751 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:15:40,755 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2730 sec 2024-11-09T21:15:41,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T21:15:41,639 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:41,647 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1487a66f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,647 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1081c072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,650 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,651 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55982, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2095feb2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,654 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,662 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:15:41,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35b1cb83 2024-11-09T21:15:41,666 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:15:41,668 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55992, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:15:41,673 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50d3f2f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,673 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,673 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,674 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,675 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f535689, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,676 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,677 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,678 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,680 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46ad9e2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,680 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,680 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,681 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f9668b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,683 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,684 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,685 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,686 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50c5f854, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,687 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6308d33d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,687 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,687 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,688 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bd1ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,692 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56042, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d903666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,694 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a822d7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,694 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,694 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,696 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@396be81f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,697 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,699 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d66170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,701 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7813eeba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,702 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@168b5662, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,704 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,705 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1853672d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,707 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f66d26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,707 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,707 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,708 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,708 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@562d1bb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,710 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56114, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,711 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69062f3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@225d1960, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,712 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,712 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,713 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cfb2098, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,714 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,715 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,716 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@424756bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@93b4e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,718 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,719 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,719 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2045255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,721 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,722 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c0a534c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,723 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7449cb3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,723 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:15:41,723 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:15:41,724 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:15:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:15:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:15:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69746154, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,725 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:15:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:15:41,726 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:15:41,727 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:15:41,728 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fb166ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:15:41,731 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,731 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,731 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,732 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,733 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,734 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,735 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47702, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,738 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-09T21:15:41,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,740 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,741 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,742 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47726, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,743 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:41,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:15:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47742, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,745 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:41,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,755 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,760 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,760 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:15:41,762 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,764 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,764 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:15:41,765 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:15:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,767 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:15:41,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:15:41,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:15:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:41,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:41,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:41,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:41,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:41,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:41,836 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:15:41,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T21:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:41,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d0733887c9d94b509b196e4e0d485015 is 50, key is test_row_0/A:col10/1731186941817/Put/seqid=0 2024-11-09T21:15:41,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:41,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:41,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:41,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741837_1013 (size=12001) 2024-11-09T21:15:41,971 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d0733887c9d94b509b196e4e0d485015 2024-11-09T21:15:41,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:41,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186941921, max timestamp=1731186941967Num uniques 859; 2024-11-09T21:15:41,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:41,997 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186941921, max timestamp=1731186941983Num uniques 859; 2024-11-09T21:15:42,004 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:42,006 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731186941921, max timestamp=1731186941967Num uniques 859; 2024-11-09T21:15:42,007 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T21:15:42,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:15:42,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/ab4eb4bf6e8b4467a459e73497c5f28e is 50, key is test_row_0/B:col10/1731186941817/Put/seqid=0 2024-11-09T21:15:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187002057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187002054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741838_1014 (size=12001) 2024-11-09T21:15:42,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/ab4eb4bf6e8b4467a459e73497c5f28e 2024-11-09T21:15:42,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187002072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187002071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187002073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,129 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T21:15:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,133 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,134 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,136 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/3a18f9be23bf46938682d9e2adada014 is 50, key is test_row_0/C:col10/1731186941817/Put/seqid=0 2024-11-09T21:15:42,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741839_1015 (size=12001) 2024-11-09T21:15:42,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/3a18f9be23bf46938682d9e2adada014 2024-11-09T21:15:42,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d0733887c9d94b509b196e4e0d485015 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d0733887c9d94b509b196e4e0d485015 2024-11-09T21:15:42,224 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d0733887c9d94b509b196e4e0d485015, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T21:15:42,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/ab4eb4bf6e8b4467a459e73497c5f28e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ab4eb4bf6e8b4467a459e73497c5f28e 2024-11-09T21:15:42,253 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ab4eb4bf6e8b4467a459e73497c5f28e, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T21:15:42,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/3a18f9be23bf46938682d9e2adada014 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3a18f9be23bf46938682d9e2adada014 2024-11-09T21:15:42,269 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3a18f9be23bf46938682d9e2adada014, entries=150, sequenceid=12, filesize=11.7 K 2024-11-09T21:15:42,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for ac01e9e325a29ffdc19c65c95fe0a1df in 456ms, sequenceid=12, compaction requested=false 2024-11-09T21:15:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T21:15:42,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:42,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:42,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=133.78 KB heapSize=384.07 KB 2024-11-09T21:15:42,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:42,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:42,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:42,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:42,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:42,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:42,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T21:15:42,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:42,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/57a3de26846c444aa01e245f69fc1cbe is 50, key is test_row_0/A:col10/1731186942065/Put/seqid=0 2024-11-09T21:15:42,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741840_1016 (size=14341) 2024-11-09T21:15:42,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/57a3de26846c444aa01e245f69fc1cbe 2024-11-09T21:15:42,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187002328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187002348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187002356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,365 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:15:42,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c6a15fc74f50433da7f7db2b5644250b is 50, key is test_row_0/B:col10/1731186942065/Put/seqid=0 2024-11-09T21:15:42,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741841_1017 (size=12001) 2024-11-09T21:15:42,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c6a15fc74f50433da7f7db2b5644250b 2024-11-09T21:15:42,415 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187002360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187002365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/ab77949e1a124651983d3bbaf8911908 is 50, key is test_row_0/C:col10/1731186942065/Put/seqid=0 2024-11-09T21:15:42,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741842_1018 (size=12001) 2024-11-09T21:15:42,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T21:15:42,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:42,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/ab77949e1a124651983d3bbaf8911908 2024-11-09T21:15:42,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/57a3de26846c444aa01e245f69fc1cbe as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/57a3de26846c444aa01e245f69fc1cbe 2024-11-09T21:15:42,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/57a3de26846c444aa01e245f69fc1cbe, entries=200, sequenceid=49, filesize=14.0 K 2024-11-09T21:15:42,509 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c6a15fc74f50433da7f7db2b5644250b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c6a15fc74f50433da7f7db2b5644250b 2024-11-09T21:15:42,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187002471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187002478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187002479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,518 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187002528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c6a15fc74f50433da7f7db2b5644250b, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T21:15:42,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187002528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,536 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/ab77949e1a124651983d3bbaf8911908 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ab77949e1a124651983d3bbaf8911908 2024-11-09T21:15:42,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ab77949e1a124651983d3bbaf8911908, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T21:15:42,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~133.78 KB/136995, heapSize ~384.02 KB/393240, currentSize=53.67 KB/54960 for ac01e9e325a29ffdc19c65c95fe0a1df in 263ms, sequenceid=49, compaction requested=false 2024-11-09T21:15:42,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:42,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-09T21:15:42,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:42,615 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:15:42,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:42,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:42,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:42,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/652861cfdb874117bb465ae5eee50a9e is 50, key is test_row_0/A:col10/1731186942355/Put/seqid=0 2024-11-09T21:15:42,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741843_1019 (size=9657) 2024-11-09T21:15:42,656 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/652861cfdb874117bb465ae5eee50a9e 2024-11-09T21:15:42,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/99fb77982e2d43758d36022812ef222c is 50, key is test_row_0/B:col10/1731186942355/Put/seqid=0 2024-11-09T21:15:42,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741844_1020 (size=9657) 2024-11-09T21:15:42,721 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/99fb77982e2d43758d36022812ef222c 2024-11-09T21:15:42,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:42,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:42,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/acbbc82559a940719ca87fdd645edb1f is 50, key is test_row_0/C:col10/1731186942355/Put/seqid=0 2024-11-09T21:15:42,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741845_1021 (size=9657) 2024-11-09T21:15:42,801 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186942791, max timestamp=1731186942796Num uniques 859; 2024-11-09T21:15:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186942791, max timestamp=1731186942796Num uniques 859; 2024-11-09T21:15:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:42,802 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186942791, max timestamp=1731186942796Num uniques 859; 2024-11-09T21:15:42,803 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:42,863 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187002849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187002852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,865 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187002859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,867 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187002859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,868 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187002862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:15:42,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187002977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187002977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187002978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187002978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,981 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:42,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187002978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:42,983 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187003187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187003188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/acbbc82559a940719ca87fdd645edb1f 2024-11-09T21:15:43,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,192 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187003189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187003189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187003191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/652861cfdb874117bb465ae5eee50a9e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/652861cfdb874117bb465ae5eee50a9e 2024-11-09T21:15:43,222 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/652861cfdb874117bb465ae5eee50a9e, entries=100, sequenceid=60, filesize=9.4 K 2024-11-09T21:15:43,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/99fb77982e2d43758d36022812ef222c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/99fb77982e2d43758d36022812ef222c 2024-11-09T21:15:43,245 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/99fb77982e2d43758d36022812ef222c, entries=100, sequenceid=60, filesize=9.4 K 2024-11-09T21:15:43,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/acbbc82559a940719ca87fdd645edb1f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/acbbc82559a940719ca87fdd645edb1f 2024-11-09T21:15:43,260 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/acbbc82559a940719ca87fdd645edb1f, entries=100, sequenceid=60, filesize=9.4 K 2024-11-09T21:15:43,263 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for ac01e9e325a29ffdc19c65c95fe0a1df in 647ms, sequenceid=60, compaction requested=true 2024-11-09T21:15:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:43,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-09T21:15:43,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-09T21:15:43,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-09T21:15:43,274 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5200 sec 2024-11-09T21:15:43,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.5510 sec 2024-11-09T21:15:43,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T21:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:43,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:43,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:43,522 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/8058b1be20cf482b8650bc252b8a353a is 50, key is test_row_0/A:col10/1731186943504/Put/seqid=0 2024-11-09T21:15:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187003522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187003525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187003527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,543 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187003529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,544 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187003542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,548 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741846_1022 (size=14341) 2024-11-09T21:15:43,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/8058b1be20cf482b8650bc252b8a353a 2024-11-09T21:15:43,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c4d97f15708f4a2daa35987fe346ba90 is 50, key is test_row_0/B:col10/1731186943504/Put/seqid=0 2024-11-09T21:15:43,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741847_1023 (size=12001) 2024-11-09T21:15:43,599 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c4d97f15708f4a2daa35987fe346ba90 2024-11-09T21:15:43,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c2ae2be3d4ce405cafc304801fc66056 is 50, key is test_row_0/C:col10/1731186943504/Put/seqid=0 2024-11-09T21:15:43,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741848_1024 (size=12001) 2024-11-09T21:15:43,633 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c2ae2be3d4ce405cafc304801fc66056 2024-11-09T21:15:43,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/8058b1be20cf482b8650bc252b8a353a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/8058b1be20cf482b8650bc252b8a353a 2024-11-09T21:15:43,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187003647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,651 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,652 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187003648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187003648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187003648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,654 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,655 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187003657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,665 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/8058b1be20cf482b8650bc252b8a353a, entries=200, sequenceid=97, filesize=14.0 K 2024-11-09T21:15:43,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c4d97f15708f4a2daa35987fe346ba90 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c4d97f15708f4a2daa35987fe346ba90 2024-11-09T21:15:43,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c4d97f15708f4a2daa35987fe346ba90, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:15:43,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c2ae2be3d4ce405cafc304801fc66056 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c2ae2be3d4ce405cafc304801fc66056 2024-11-09T21:15:43,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c2ae2be3d4ce405cafc304801fc66056, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:15:43,708 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for ac01e9e325a29ffdc19c65c95fe0a1df in 201ms, sequenceid=97, compaction requested=true 2024-11-09T21:15:43,708 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:43,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:43,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:43,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:43,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:43,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:43,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:43,713 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:43,713 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:43,717 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50340 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:43,717 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:43,719 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:43,720 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:43,720 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:43,720 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:43,720 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ab4eb4bf6e8b4467a459e73497c5f28e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c6a15fc74f50433da7f7db2b5644250b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/99fb77982e2d43758d36022812ef222c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c4d97f15708f4a2daa35987fe346ba90] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=44.6 K 2024-11-09T21:15:43,720 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d0733887c9d94b509b196e4e0d485015, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/57a3de26846c444aa01e245f69fc1cbe, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/652861cfdb874117bb465ae5eee50a9e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/8058b1be20cf482b8650bc252b8a353a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=49.2 K 2024-11-09T21:15:43,724 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0733887c9d94b509b196e4e0d485015, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731186941759 2024-11-09T21:15:43,724 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ab4eb4bf6e8b4467a459e73497c5f28e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731186941759 2024-11-09T21:15:43,725 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a15fc74f50433da7f7db2b5644250b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731186942065 2024-11-09T21:15:43,727 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 99fb77982e2d43758d36022812ef222c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731186942349 2024-11-09T21:15:43,727 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57a3de26846c444aa01e245f69fc1cbe, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731186942052 2024-11-09T21:15:43,728 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 652861cfdb874117bb465ae5eee50a9e, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731186942349 2024-11-09T21:15:43,728 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c4d97f15708f4a2daa35987fe346ba90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186942852 2024-11-09T21:15:43,732 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8058b1be20cf482b8650bc252b8a353a, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186942848 2024-11-09T21:15:43,781 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T21:15:43,786 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#12 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:43,787 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/a1245dde58084cca87b3f98006f287d1 is 50, key is test_row_0/A:col10/1731186943504/Put/seqid=0 2024-11-09T21:15:43,801 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#13 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:43,802 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cb0f7681472849a5a7d3c14f34f62641 is 50, key is test_row_0/B:col10/1731186943504/Put/seqid=0 2024-11-09T21:15:43,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741849_1025 (size=12139) 2024-11-09T21:15:43,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741850_1026 (size=12139) 2024-11-09T21:15:43,854 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cb0f7681472849a5a7d3c14f34f62641 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cb0f7681472849a5a7d3c14f34f62641 2024-11-09T21:15:43,854 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/a1245dde58084cca87b3f98006f287d1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/a1245dde58084cca87b3f98006f287d1 2024-11-09T21:15:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:43,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:15:43,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:43,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:43,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:43,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:43,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:43,865 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:43,874 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d4c23e142d5d4befa1f69ec95a0ca297 is 50, key is test_row_0/A:col10/1731186943516/Put/seqid=0 2024-11-09T21:15:43,886 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into a1245dde58084cca87b3f98006f287d1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:43,886 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:43,886 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into cb0f7681472849a5a7d3c14f34f62641(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:43,887 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:43,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-09T21:15:43,887 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:43,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:43,889 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=12, startTime=1731186943713; duration=0sec 2024-11-09T21:15:43,889 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=12, startTime=1731186943709; duration=0sec 2024-11-09T21:15:43,889 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:43,890 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:43,890 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:43,890 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:43,890 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:43,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-09T21:15:43,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T21:15:43,894 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:43,895 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45660 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:43,895 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:43,896 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:43,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:43,896 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:43,896 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3a18f9be23bf46938682d9e2adada014, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ab77949e1a124651983d3bbaf8911908, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/acbbc82559a940719ca87fdd645edb1f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c2ae2be3d4ce405cafc304801fc66056] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=44.6 K 2024-11-09T21:15:43,897 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3a18f9be23bf46938682d9e2adada014, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731186941759 2024-11-09T21:15:43,898 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ab77949e1a124651983d3bbaf8911908, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731186942065 2024-11-09T21:15:43,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting acbbc82559a940719ca87fdd645edb1f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731186942349 2024-11-09T21:15:43,901 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c2ae2be3d4ce405cafc304801fc66056, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186942852 2024-11-09T21:15:43,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741851_1027 (size=12001) 2024-11-09T21:15:43,914 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d4c23e142d5d4befa1f69ec95a0ca297 2024-11-09T21:15:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187003925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,940 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187003928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187003932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187003933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/75ef6062863c4de7bc7a997262fabbb2 is 50, key is test_row_0/B:col10/1731186943516/Put/seqid=0 2024-11-09T21:15:43,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:43,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187003939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,944 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:43,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:43,951 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:43,952 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/7288d2a2e5e244da9d807f1d14842776 is 50, key is test_row_0/C:col10/1731186943504/Put/seqid=0 2024-11-09T21:15:43,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741852_1028 (size=12001) 2024-11-09T21:15:43,979 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/75ef6062863c4de7bc7a997262fabbb2 2024-11-09T21:15:43,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T21:15:43,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741853_1029 (size=12139) 2024-11-09T21:15:44,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/7c77915d80834933a3c3ac951955c613 is 50, key is test_row_0/C:col10/1731186943516/Put/seqid=0 2024-11-09T21:15:44,011 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/7288d2a2e5e244da9d807f1d14842776 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7288d2a2e5e244da9d807f1d14842776 2024-11-09T21:15:44,029 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into 7288d2a2e5e244da9d807f1d14842776(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:44,029 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:44,029 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=12, startTime=1731186943713; duration=0sec 2024-11-09T21:15:44,030 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:44,030 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:44,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741854_1030 (size=12001) 2024-11-09T21:15:44,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=109 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/7c77915d80834933a3c3ac951955c613 2024-11-09T21:15:44,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T21:15:44,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:44,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:44,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:44,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187004046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187004047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:44,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187004047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,055 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187004057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,062 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187004057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d4c23e142d5d4befa1f69ec95a0ca297 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d4c23e142d5d4befa1f69ec95a0ca297 2024-11-09T21:15:44,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d4c23e142d5d4befa1f69ec95a0ca297, entries=150, sequenceid=109, filesize=11.7 K 2024-11-09T21:15:44,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/75ef6062863c4de7bc7a997262fabbb2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/75ef6062863c4de7bc7a997262fabbb2 2024-11-09T21:15:44,108 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/75ef6062863c4de7bc7a997262fabbb2, entries=150, sequenceid=109, filesize=11.7 K 2024-11-09T21:15:44,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/7c77915d80834933a3c3ac951955c613 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7c77915d80834933a3c3ac951955c613 2024-11-09T21:15:44,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7c77915d80834933a3c3ac951955c613, entries=150, sequenceid=109, filesize=11.7 K 2024-11-09T21:15:44,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for ac01e9e325a29ffdc19c65c95fe0a1df in 264ms, sequenceid=109, compaction requested=false 2024-11-09T21:15:44,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T21:15:44,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-09T21:15:44,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:44,209 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:44,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:44,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:44,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:44,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/08e517ea8d7d41799c47d42aaa404d53 is 50, key is test_row_0/A:col10/1731186943926/Put/seqid=0 2024-11-09T21:15:44,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741855_1031 (size=12101) 2024-11-09T21:15:44,303 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/08e517ea8d7d41799c47d42aaa404d53 2024-11-09T21:15:44,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/38838d90cdb8443eae3fac2e9c683dd2 is 50, key is test_row_0/B:col10/1731186943926/Put/seqid=0 2024-11-09T21:15:44,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741856_1032 (size=12101) 2024-11-09T21:15:44,354 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187004292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187004294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,356 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187004355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,361 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187004355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187004368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,371 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187004466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187004467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187004467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187004478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187004478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T21:15:44,566 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:15:44,640 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-09T21:15:44,640 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-09T21:15:44,643 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T21:15:44,645 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-09T21:15:44,645 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-09T21:15:44,646 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-09T21:15:44,647 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T21:15:44,647 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T21:15:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187004677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187004677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187004677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,683 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187004687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187004687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:44,692 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:44,693 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:44,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/38838d90cdb8443eae3fac2e9c683dd2 2024-11-09T21:15:44,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/2747bdb68eee4f8c8db199416373191a is 50, key is test_row_0/C:col10/1731186943926/Put/seqid=0 2024-11-09T21:15:44,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741857_1033 (size=12101) 2024-11-09T21:15:44,759 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/2747bdb68eee4f8c8db199416373191a 2024-11-09T21:15:44,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/08e517ea8d7d41799c47d42aaa404d53 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/08e517ea8d7d41799c47d42aaa404d53 2024-11-09T21:15:44,794 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/08e517ea8d7d41799c47d42aaa404d53, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:15:44,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/38838d90cdb8443eae3fac2e9c683dd2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/38838d90cdb8443eae3fac2e9c683dd2 2024-11-09T21:15:44,813 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/38838d90cdb8443eae3fac2e9c683dd2, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:15:44,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/2747bdb68eee4f8c8db199416373191a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/2747bdb68eee4f8c8db199416373191a 2024-11-09T21:15:44,825 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/2747bdb68eee4f8c8db199416373191a, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:15:44,827 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=73.80 KB/75570 for ac01e9e325a29ffdc19c65c95fe0a1df in 618ms, sequenceid=132, compaction requested=true 2024-11-09T21:15:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:44,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-09T21:15:44,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-09T21:15:44,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-09T21:15:44,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 933 msec 2024-11-09T21:15:44,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 946 msec 2024-11-09T21:15:44,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:44,993 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:15:44,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:44,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:44,993 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:44,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:44,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:44,994 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:45,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/ffa65120ddfd45d3b6eb7ea341807813 is 50, key is test_row_0/A:col10/1731186944273/Put/seqid=0 2024-11-09T21:15:45,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-09T21:15:45,028 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:45,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:45,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-09T21:15:45,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187005021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,034 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187005021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,034 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,034 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187005028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:45,037 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:45,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:45,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187005030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187005033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741858_1034 (size=12151) 2024-11-09T21:15:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,044 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187005136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,138 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:45,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187005148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187005148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187005149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187005150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,154 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,155 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,156 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:45,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:45,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,347 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187005347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,351 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:45,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187005356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,362 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187005368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,372 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187005368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187005369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,373 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,374 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,447 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/ffa65120ddfd45d3b6eb7ea341807813 2024-11-09T21:15:45,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c33b9db369934af1b65dc7c799b2fe57 is 50, key is test_row_0/B:col10/1731186944273/Put/seqid=0 2024-11-09T21:15:45,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741859_1035 (size=12151) 2024-11-09T21:15:45,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:45,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187005656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:45,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187005667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,669 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187005676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187005677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:45,681 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187005677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:45,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:45,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:45,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:45,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c33b9db369934af1b65dc7c799b2fe57 2024-11-09T21:15:45,930 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/a1b40204674745c6aa7d31f8ee4f1804 is 50, key is test_row_0/C:col10/1731186944273/Put/seqid=0 2024-11-09T21:15:45,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741860_1036 (size=12151) 2024-11-09T21:15:45,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:45,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:46,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:46,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:46,180 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:46,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187006177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:46,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:46,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187006178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:46,182 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,183 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:46,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:46,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187006187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:46,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:46,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187006187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,195 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:46,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:46,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187006197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:46,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:46,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:46,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:46,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=148 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/a1b40204674745c6aa7d31f8ee4f1804 2024-11-09T21:15:46,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/ffa65120ddfd45d3b6eb7ea341807813 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/ffa65120ddfd45d3b6eb7ea341807813 2024-11-09T21:15:46,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/ffa65120ddfd45d3b6eb7ea341807813, entries=150, sequenceid=148, filesize=11.9 K 2024-11-09T21:15:46,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c33b9db369934af1b65dc7c799b2fe57 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c33b9db369934af1b65dc7c799b2fe57 2024-11-09T21:15:46,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c33b9db369934af1b65dc7c799b2fe57, entries=150, sequenceid=148, filesize=11.9 K 2024-11-09T21:15:46,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/a1b40204674745c6aa7d31f8ee4f1804 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/a1b40204674745c6aa7d31f8ee4f1804 2024-11-09T21:15:46,392 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/a1b40204674745c6aa7d31f8ee4f1804, entries=150, sequenceid=148, filesize=11.9 K 2024-11-09T21:15:46,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ac01e9e325a29ffdc19c65c95fe0a1df in 1402ms, sequenceid=148, compaction requested=true 2024-11-09T21:15:46,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:46,395 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:46,395 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:46,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:46,397 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:46,397 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:46,397 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,398 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/a1245dde58084cca87b3f98006f287d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d4c23e142d5d4befa1f69ec95a0ca297, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/08e517ea8d7d41799c47d42aaa404d53, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/ffa65120ddfd45d3b6eb7ea341807813] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=47.3 K 2024-11-09T21:15:46,398 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:46,398 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:46,398 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,398 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cb0f7681472849a5a7d3c14f34f62641, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/75ef6062863c4de7bc7a997262fabbb2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/38838d90cdb8443eae3fac2e9c683dd2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c33b9db369934af1b65dc7c799b2fe57] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=47.3 K 2024-11-09T21:15:46,398 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1245dde58084cca87b3f98006f287d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186942852 2024-11-09T21:15:46,399 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c23e142d5d4befa1f69ec95a0ca297, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731186943516 2024-11-09T21:15:46,399 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting cb0f7681472849a5a7d3c14f34f62641, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186942852 2024-11-09T21:15:46,400 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08e517ea8d7d41799c47d42aaa404d53, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731186943926 2024-11-09T21:15:46,401 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 75ef6062863c4de7bc7a997262fabbb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731186943516 2024-11-09T21:15:46,402 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 38838d90cdb8443eae3fac2e9c683dd2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731186943926 2024-11-09T21:15:46,402 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa65120ddfd45d3b6eb7ea341807813, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731186944273 2024-11-09T21:15:46,404 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c33b9db369934af1b65dc7c799b2fe57, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731186944273 2024-11-09T21:15:46,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-09T21:15:46,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T21:15:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:46,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:46,440 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#25 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:46,442 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#24 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:46,442 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/e6bfac905bd14b818347a9762b4905af is 50, key is test_row_0/B:col10/1731186944273/Put/seqid=0 2024-11-09T21:15:46,443 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/65226c0956114d529d91805a62ebca20 is 50, key is test_row_0/A:col10/1731186944273/Put/seqid=0 2024-11-09T21:15:46,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/01c144b4000f4cb0a7439ea8b2a9841b is 50, key is test_row_0/A:col10/1731186945027/Put/seqid=0 2024-11-09T21:15:46,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741862_1038 (size=12425) 2024-11-09T21:15:46,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741863_1039 (size=12151) 2024-11-09T21:15:46,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741861_1037 (size=12425) 2024-11-09T21:15:46,492 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/65226c0956114d529d91805a62ebca20 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/65226c0956114d529d91805a62ebca20 2024-11-09T21:15:46,497 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/e6bfac905bd14b818347a9762b4905af as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/e6bfac905bd14b818347a9762b4905af 2024-11-09T21:15:46,508 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into 65226c0956114d529d91805a62ebca20(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:46,508 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:46,508 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=12, startTime=1731186946394; duration=0sec 2024-11-09T21:15:46,508 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:46,508 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:46,508 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:46,510 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into e6bfac905bd14b818347a9762b4905af(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:46,510 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:46,510 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=12, startTime=1731186946395; duration=0sec 2024-11-09T21:15:46,510 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:46,510 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:46,511 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48392 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:46,511 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:46,511 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:46,511 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7288d2a2e5e244da9d807f1d14842776, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7c77915d80834933a3c3ac951955c613, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/2747bdb68eee4f8c8db199416373191a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/a1b40204674745c6aa7d31f8ee4f1804] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=47.3 K 2024-11-09T21:15:46,512 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7288d2a2e5e244da9d807f1d14842776, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186942852 2024-11-09T21:15:46,512 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c77915d80834933a3c3ac951955c613, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=109, earliestPutTs=1731186943516 2024-11-09T21:15:46,513 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2747bdb68eee4f8c8db199416373191a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731186943926 2024-11-09T21:15:46,515 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a1b40204674745c6aa7d31f8ee4f1804, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731186944273 2024-11-09T21:15:46,551 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#27 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:46,552 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/b49f908889c34492a82bd5c931fa112d is 50, key is test_row_0/C:col10/1731186944273/Put/seqid=0 2024-11-09T21:15:46,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741864_1040 (size=12425) 2024-11-09T21:15:46,570 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/b49f908889c34492a82bd5c931fa112d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/b49f908889c34492a82bd5c931fa112d 2024-11-09T21:15:46,590 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into b49f908889c34492a82bd5c931fa112d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:46,590 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:46,590 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=12, startTime=1731186946395; duration=0sec 2024-11-09T21:15:46,590 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:46,590 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:46,883 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/01c144b4000f4cb0a7439ea8b2a9841b 2024-11-09T21:15:46,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/4f24b58f98e24ae6918c02724fd2839a is 50, key is test_row_0/B:col10/1731186945027/Put/seqid=0 2024-11-09T21:15:46,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741865_1041 (size=12151) 2024-11-09T21:15:46,924 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/4f24b58f98e24ae6918c02724fd2839a 2024-11-09T21:15:46,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/65b6b9d8e5bf485da1ee1ccdf80714e5 is 50, key is test_row_0/C:col10/1731186945027/Put/seqid=0 2024-11-09T21:15:46,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741866_1042 (size=12151) 2024-11-09T21:15:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:47,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187007218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187007223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187007223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187007229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187007231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,244 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187007337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,340 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187007337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187007346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187007346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187007347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,352 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/65b6b9d8e5bf485da1ee1ccdf80714e5 2024-11-09T21:15:47,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/01c144b4000f4cb0a7439ea8b2a9841b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/01c144b4000f4cb0a7439ea8b2a9841b 2024-11-09T21:15:47,389 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/01c144b4000f4cb0a7439ea8b2a9841b, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T21:15:47,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/4f24b58f98e24ae6918c02724fd2839a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/4f24b58f98e24ae6918c02724fd2839a 2024-11-09T21:15:47,412 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/4f24b58f98e24ae6918c02724fd2839a, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T21:15:47,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/65b6b9d8e5bf485da1ee1ccdf80714e5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/65b6b9d8e5bf485da1ee1ccdf80714e5 2024-11-09T21:15:47,432 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/65b6b9d8e5bf485da1ee1ccdf80714e5, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T21:15:47,434 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for ac01e9e325a29ffdc19c65c95fe0a1df in 997ms, sequenceid=164, compaction requested=false 2024-11-09T21:15:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:47,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-09T21:15:47,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-09T21:15:47,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-09T21:15:47,439 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3990 sec 2024-11-09T21:15:47,444 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4120 sec 2024-11-09T21:15:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:47,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:15:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:47,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:47,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:47,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:47,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:47,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/7492cc05e72b4839978740478226e014 is 50, key is test_row_0/A:col10/1731186947215/Put/seqid=0 2024-11-09T21:15:47,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741867_1043 (size=16931) 2024-11-09T21:15:47,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187007620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,636 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187007627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187007630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187007631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,638 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,639 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187007636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187007747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,749 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187007748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187007748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187007747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,751 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187007748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,752 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187007956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187007956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,960 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187007956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187007957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:47,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187007957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:47,962 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:47,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/7492cc05e72b4839978740478226e014 2024-11-09T21:15:47,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/34810630094841dbb389f9d0f6908755 is 50, key is test_row_0/B:col10/1731186947215/Put/seqid=0 2024-11-09T21:15:48,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741868_1044 (size=12151) 2024-11-09T21:15:48,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187008266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187008266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,269 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187008266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187008267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187008267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,270 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,271 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/34810630094841dbb389f9d0f6908755 2024-11-09T21:15:48,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/3ed51fd076c94dd1bf659750633da8fb is 50, key is test_row_0/C:col10/1731186947215/Put/seqid=0 2024-11-09T21:15:48,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741869_1045 (size=12151) 2024-11-09T21:15:48,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187008776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187008776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187008776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187008776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,782 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:48,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187008787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:48,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:48,788 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:48,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/3ed51fd076c94dd1bf659750633da8fb 2024-11-09T21:15:48,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/7492cc05e72b4839978740478226e014 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7492cc05e72b4839978740478226e014 2024-11-09T21:15:48,851 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7492cc05e72b4839978740478226e014, entries=250, sequenceid=185, filesize=16.5 K 2024-11-09T21:15:48,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/34810630094841dbb389f9d0f6908755 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/34810630094841dbb389f9d0f6908755 2024-11-09T21:15:48,861 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/34810630094841dbb389f9d0f6908755, entries=150, sequenceid=185, filesize=11.9 K 2024-11-09T21:15:48,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/3ed51fd076c94dd1bf659750633da8fb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3ed51fd076c94dd1bf659750633da8fb 2024-11-09T21:15:48,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3ed51fd076c94dd1bf659750633da8fb, entries=150, sequenceid=185, filesize=11.9 K 2024-11-09T21:15:48,872 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for ac01e9e325a29ffdc19c65c95fe0a1df in 1322ms, sequenceid=185, compaction requested=true 2024-11-09T21:15:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:48,872 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:48,872 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:48,872 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:48,873 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:48,874 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:48,874 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41507 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:48,874 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:48,874 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:48,874 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:48,874 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:48,874 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/e6bfac905bd14b818347a9762b4905af, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/4f24b58f98e24ae6918c02724fd2839a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/34810630094841dbb389f9d0f6908755] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=35.9 K 2024-11-09T21:15:48,874 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/65226c0956114d529d91805a62ebca20, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/01c144b4000f4cb0a7439ea8b2a9841b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7492cc05e72b4839978740478226e014] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=40.5 K 2024-11-09T21:15:48,875 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e6bfac905bd14b818347a9762b4905af, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731186944273 2024-11-09T21:15:48,875 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65226c0956114d529d91805a62ebca20, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731186944273 2024-11-09T21:15:48,875 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f24b58f98e24ae6918c02724fd2839a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731186945008 2024-11-09T21:15:48,875 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01c144b4000f4cb0a7439ea8b2a9841b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731186945008 2024-11-09T21:15:48,876 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 34810630094841dbb389f9d0f6908755, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731186947211 2024-11-09T21:15:48,876 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7492cc05e72b4839978740478226e014, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731186947211 2024-11-09T21:15:48,889 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#33 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:48,890 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/708dd1f8e3524ed9b7323c3162d75e25 is 50, key is test_row_0/B:col10/1731186947215/Put/seqid=0 2024-11-09T21:15:48,894 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#34 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:48,895 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1d5f2472d2354a6a83c2787ee91ddb3e is 50, key is test_row_0/A:col10/1731186947215/Put/seqid=0 2024-11-09T21:15:48,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741871_1047 (size=12527) 2024-11-09T21:15:48,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741870_1046 (size=12527) 2024-11-09T21:15:48,928 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/708dd1f8e3524ed9b7323c3162d75e25 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/708dd1f8e3524ed9b7323c3162d75e25 2024-11-09T21:15:48,930 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1d5f2472d2354a6a83c2787ee91ddb3e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d5f2472d2354a6a83c2787ee91ddb3e 2024-11-09T21:15:48,942 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into 708dd1f8e3524ed9b7323c3162d75e25(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:48,942 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:48,942 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=13, startTime=1731186948872; duration=0sec 2024-11-09T21:15:48,942 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into 1d5f2472d2354a6a83c2787ee91ddb3e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:48,942 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:48,942 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=13, startTime=1731186948872; duration=0sec 2024-11-09T21:15:48,942 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:48,943 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:48,943 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:48,943 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:48,943 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:48,945 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:48,945 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:48,945 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:48,945 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/b49f908889c34492a82bd5c931fa112d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/65b6b9d8e5bf485da1ee1ccdf80714e5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3ed51fd076c94dd1bf659750633da8fb] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=35.9 K 2024-11-09T21:15:48,946 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b49f908889c34492a82bd5c931fa112d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=148, earliestPutTs=1731186944273 2024-11-09T21:15:48,946 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 65b6b9d8e5bf485da1ee1ccdf80714e5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731186945008 2024-11-09T21:15:48,947 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ed51fd076c94dd1bf659750633da8fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731186947211 2024-11-09T21:15:48,961 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:48,962 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e3139bd5a22f4797a9eb8fe23c13903a is 50, key is test_row_0/C:col10/1731186947215/Put/seqid=0 2024-11-09T21:15:48,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741872_1048 (size=12527) 2024-11-09T21:15:48,982 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e3139bd5a22f4797a9eb8fe23c13903a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e3139bd5a22f4797a9eb8fe23c13903a 2024-11-09T21:15:48,997 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into e3139bd5a22f4797a9eb8fe23c13903a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:48,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:48,997 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=13, startTime=1731186948872; duration=0sec 2024-11-09T21:15:48,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:48,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:49,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-09T21:15:49,207 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:49,207 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:49,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-09T21:15:49,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:15:49,210 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:49,211 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:49,211 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:15:49,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-09T21:15:49,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:49,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T21:15:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:49,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:49,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:49,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b230acaf2c75400bae89adcd54c1e555 is 50, key is test_row_0/A:col10/1731186947629/Put/seqid=0 2024-11-09T21:15:49,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741873_1049 (size=12151) 2024-11-09T21:15:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:15:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:49,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:49,819 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b230acaf2c75400bae89adcd54c1e555 2024-11-09T21:15:49,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187009818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187009820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,836 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187009822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:15:49,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187009829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187009834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,839 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cadf4ccdc8b34558b5f5c43977caef86 is 50, key is test_row_0/B:col10/1731186947629/Put/seqid=0 2024-11-09T21:15:49,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741874_1050 (size=12151) 2024-11-09T21:15:49,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187009936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187009947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187009947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187009947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187009947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,952 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:49,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187010147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187010157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187010157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187010158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,161 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187010158, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cadf4ccdc8b34558b5f5c43977caef86 2024-11-09T21:15:50,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/d28f1709234647509fc43ba724dd7bd4 is 50, key is test_row_0/C:col10/1731186947629/Put/seqid=0 2024-11-09T21:15:50,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:15:50,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741875_1051 (size=12151) 2024-11-09T21:15:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187010456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187010466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187010466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,472 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187010467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,479 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:50,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187010477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:50,480 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:50,766 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/d28f1709234647509fc43ba724dd7bd4 2024-11-09T21:15:50,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b230acaf2c75400bae89adcd54c1e555 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b230acaf2c75400bae89adcd54c1e555 2024-11-09T21:15:50,795 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b230acaf2c75400bae89adcd54c1e555, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T21:15:50,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cadf4ccdc8b34558b5f5c43977caef86 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cadf4ccdc8b34558b5f5c43977caef86 2024-11-09T21:15:50,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cadf4ccdc8b34558b5f5c43977caef86, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T21:15:50,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/d28f1709234647509fc43ba724dd7bd4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/d28f1709234647509fc43ba724dd7bd4 2024-11-09T21:15:50,823 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/d28f1709234647509fc43ba724dd7bd4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T21:15:50,825 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for ac01e9e325a29ffdc19c65c95fe0a1df in 1460ms, sequenceid=202, compaction requested=false 2024-11-09T21:15:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-09T21:15:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-09T21:15:50,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-09T21:15:50,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6160 sec 2024-11-09T21:15:50,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.6280 sec 2024-11-09T21:15:51,028 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731186949802, max timestamp=1731186950991Num uniques 859; 2024-11-09T21:15:51,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:51,031 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731186949802, max timestamp=1731186950988Num uniques 859; 2024-11-09T21:15:51,032 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T21:15:51,029 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731186949802, max timestamp=1731186950991Num uniques 859; 2024-11-09T21:15:51,035 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:51,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=37.62 KB heapSize=95.40 KB 2024-11-09T21:15:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:51,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:51,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:51,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:51,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d20ac74e94b948acaf2fa8f4a60d4ac6 is 50, key is test_row_0/A:col10/1731186950968/Put/seqid=0 2024-11-09T21:15:51,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741876_1052 (size=14541) 2024-11-09T21:15:51,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d20ac74e94b948acaf2fa8f4a60d4ac6 2024-11-09T21:15:51,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186951058, max timestamp=1731186951104Num uniques 859; 2024-11-09T21:15:51,108 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:51,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186951058, max timestamp=1731186951104Num uniques 859; 2024-11-09T21:15:51,110 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:51,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/2d576dd8a0b54a91a95741e86c302ba2 is 50, key is test_row_0/B:col10/1731186950968/Put/seqid=0 2024-11-09T21:15:51,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186951058, max timestamp=1731186951102Num uniques 859; 2024-11-09T21:15:51,118 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741877_1053 (size=12151) 2024-11-09T21:15:51,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/2d576dd8a0b54a91a95741e86c302ba2 2024-11-09T21:15:51,158 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c779c31c50ce47ca977cde1cfbd79480 is 50, key is test_row_0/C:col10/1731186950968/Put/seqid=0 2024-11-09T21:15:51,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187011163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187011165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187011167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,181 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187011169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187011172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741878_1054 (size=12151) 2024-11-09T21:15:51,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187011287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,296 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,296 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187011287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,297 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187011287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187011287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187011288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-09T21:15:51,358 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:51,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:51,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-09T21:15:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T21:15:51,363 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:51,364 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:51,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T21:15:51,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187011507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187011508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187011508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187011508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187011508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T21:15:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:51,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:51,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:51,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:51,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:51,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:51,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.26 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c779c31c50ce47ca977cde1cfbd79480 2024-11-09T21:15:51,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d20ac74e94b948acaf2fa8f4a60d4ac6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d20ac74e94b948acaf2fa8f4a60d4ac6 2024-11-09T21:15:51,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d20ac74e94b948acaf2fa8f4a60d4ac6, entries=200, sequenceid=226, filesize=14.2 K 2024-11-09T21:15:51,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/2d576dd8a0b54a91a95741e86c302ba2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/2d576dd8a0b54a91a95741e86c302ba2 2024-11-09T21:15:51,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/2d576dd8a0b54a91a95741e86c302ba2, entries=150, sequenceid=226, filesize=11.9 K 2024-11-09T21:15:51,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c779c31c50ce47ca977cde1cfbd79480 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c779c31c50ce47ca977cde1cfbd79480 2024-11-09T21:15:51,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c779c31c50ce47ca977cde1cfbd79480, entries=150, sequenceid=226, filesize=11.9 K 2024-11-09T21:15:51,632 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~51.04 KB/52265, heapSize ~135.20 KB/138440, currentSize=134.18 KB/137400 for ac01e9e325a29ffdc19c65c95fe0a1df in 596ms, sequenceid=226, compaction requested=true 2024-11-09T21:15:51,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:51,633 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:51,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:51,633 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:51,634 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:51,634 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:51,635 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:51,635 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d5f2472d2354a6a83c2787ee91ddb3e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b230acaf2c75400bae89adcd54c1e555, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d20ac74e94b948acaf2fa8f4a60d4ac6] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=38.3 K 2024-11-09T21:15:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:51,635 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:51,636 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:51,636 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d5f2472d2354a6a83c2787ee91ddb3e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731186947211 2024-11-09T21:15:51,636 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:51,636 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:51,636 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/708dd1f8e3524ed9b7323c3162d75e25, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cadf4ccdc8b34558b5f5c43977caef86, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/2d576dd8a0b54a91a95741e86c302ba2] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.0 K 2024-11-09T21:15:51,640 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b230acaf2c75400bae89adcd54c1e555, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731186947624 2024-11-09T21:15:51,640 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 708dd1f8e3524ed9b7323c3162d75e25, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731186947211 2024-11-09T21:15:51,641 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d20ac74e94b948acaf2fa8f4a60d4ac6, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731186950968 2024-11-09T21:15:51,641 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting cadf4ccdc8b34558b5f5c43977caef86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731186947624 2024-11-09T21:15:51,641 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d576dd8a0b54a91a95741e86c302ba2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731186950968 2024-11-09T21:15:51,658 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#42 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:51,659 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1d6987dc8d6f43828825af97988d27ea is 50, key is test_row_0/A:col10/1731186950968/Put/seqid=0 2024-11-09T21:15:51,669 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:51,669 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/73fa3721bd80485c8fb7e5d9f8936371 is 50, key is test_row_0/B:col10/1731186950968/Put/seqid=0 2024-11-09T21:15:51,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-09T21:15:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T21:15:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:51,677 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T21:15:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:51,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:51,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741879_1055 (size=12629) 2024-11-09T21:15:51,703 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1d6987dc8d6f43828825af97988d27ea as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d6987dc8d6f43828825af97988d27ea 2024-11-09T21:15:51,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/fafcb343dd3a42d9a14823cf2b1aca7c is 50, key is test_row_0/A:col10/1731186951164/Put/seqid=0 2024-11-09T21:15:51,712 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into 1d6987dc8d6f43828825af97988d27ea(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:51,713 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:51,713 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=13, startTime=1731186951633; duration=0sec 2024-11-09T21:15:51,713 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:51,713 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:51,713 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:51,715 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:51,715 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:51,715 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:51,715 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e3139bd5a22f4797a9eb8fe23c13903a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/d28f1709234647509fc43ba724dd7bd4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c779c31c50ce47ca977cde1cfbd79480] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.0 K 2024-11-09T21:15:51,716 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3139bd5a22f4797a9eb8fe23c13903a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731186947211 2024-11-09T21:15:51,717 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d28f1709234647509fc43ba724dd7bd4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731186947624 2024-11-09T21:15:51,718 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c779c31c50ce47ca977cde1cfbd79480, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731186950968 2024-11-09T21:15:51,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741880_1056 (size=12629) 2024-11-09T21:15:51,738 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/73fa3721bd80485c8fb7e5d9f8936371 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/73fa3721bd80485c8fb7e5d9f8936371 2024-11-09T21:15:51,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741881_1057 (size=12301) 2024-11-09T21:15:51,743 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/fafcb343dd3a42d9a14823cf2b1aca7c 2024-11-09T21:15:51,751 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into 73fa3721bd80485c8fb7e5d9f8936371(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:51,751 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:51,751 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=13, startTime=1731186951633; duration=0sec 2024-11-09T21:15:51,752 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:51,752 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:51,765 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#45 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:51,765 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/476a900f31564508920c484fdd26eb25 is 50, key is test_row_0/C:col10/1731186950968/Put/seqid=0 2024-11-09T21:15:51,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/aacec4063b824f12a90da30fbafd90de is 50, key is test_row_0/B:col10/1731186951164/Put/seqid=0 2024-11-09T21:15:51,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741882_1058 (size=12629) 2024-11-09T21:15:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:51,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:51,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741883_1059 (size=12301) 2024-11-09T21:15:51,836 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/aacec4063b824f12a90da30fbafd90de 2024-11-09T21:15:51,845 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187011836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,846 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,849 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187011840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187011840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187011842, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187011837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,852 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/ed94f44ea04347cab1366b63b4f0742b is 50, key is test_row_0/C:col10/1731186951164/Put/seqid=0 2024-11-09T21:15:51,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741884_1060 (size=12301) 2024-11-09T21:15:51,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187011957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187011957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187011957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187011957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187011957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T21:15:52,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187012166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187012167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187012167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,169 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187012167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,180 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187012176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,243 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/476a900f31564508920c484fdd26eb25 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/476a900f31564508920c484fdd26eb25 2024-11-09T21:15:52,253 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into 476a900f31564508920c484fdd26eb25(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:52,253 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:52,253 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=13, startTime=1731186951635; duration=0sec 2024-11-09T21:15:52,255 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:52,255 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:52,283 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=262 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/ed94f44ea04347cab1366b63b4f0742b 2024-11-09T21:15:52,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/fafcb343dd3a42d9a14823cf2b1aca7c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/fafcb343dd3a42d9a14823cf2b1aca7c 2024-11-09T21:15:52,314 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/fafcb343dd3a42d9a14823cf2b1aca7c, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T21:15:52,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/aacec4063b824f12a90da30fbafd90de as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/aacec4063b824f12a90da30fbafd90de 2024-11-09T21:15:52,322 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/aacec4063b824f12a90da30fbafd90de, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T21:15:52,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/ed94f44ea04347cab1366b63b4f0742b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ed94f44ea04347cab1366b63b4f0742b 2024-11-09T21:15:52,334 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ed94f44ea04347cab1366b63b4f0742b, entries=150, sequenceid=262, filesize=12.0 K 2024-11-09T21:15:52,335 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ac01e9e325a29ffdc19c65c95fe0a1df in 658ms, sequenceid=262, compaction requested=false 2024-11-09T21:15:52,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:52,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-09T21:15:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-09T21:15:52,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-09T21:15:52,340 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 974 msec 2024-11-09T21:15:52,343 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 982 msec 2024-11-09T21:15:52,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:15:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:52,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-09T21:15:52,497 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:52,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/14114e03c1ed45e8a5ad3f06546a833a is 50, key is test_row_0/A:col10/1731186952478/Put/seqid=0 2024-11-09T21:15:52,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-09T21:15:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T21:15:52,501 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:52,502 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:52,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:52,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741885_1061 (size=14741) 2024-11-09T21:15:52,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187012537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187012539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,549 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186952532, max timestamp=1731186952547Num uniques 859; 2024-11-09T21:15:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:15:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186952532, max timestamp=1731186952547Num uniques 859; 2024-11-09T21:15:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:15:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186952532, max timestamp=1731186952547Num uniques 859; 2024-11-09T21:15:52,551 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:15:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187012574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187012577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,584 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187012578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,585 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T21:15:52,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187012656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187012656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T21:15:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:52,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,668 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187012687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,706 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187012696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187012696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T21:15:52,822 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T21:15:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187012866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,874 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187012866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187012897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187012916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187012916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:52,918 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:52,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/14114e03c1ed45e8a5ad3f06546a833a 2024-11-09T21:15:52,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/f6c36c6938184c489c68d169e9be2d59 is 50, key is test_row_0/B:col10/1731186952478/Put/seqid=0 2024-11-09T21:15:52,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741886_1062 (size=12301) 2024-11-09T21:15:52,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/f6c36c6938184c489c68d169e9be2d59 2024-11-09T21:15:52,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f26c528f6f4d434d96906e005398b834 is 50, key is test_row_0/C:col10/1731186952478/Put/seqid=0 2024-11-09T21:15:52,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741887_1063 (size=12301) 2024-11-09T21:15:52,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T21:15:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:52,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:52,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T21:15:53,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T21:15:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,130 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187013186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187013186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,190 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,191 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187013206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187013226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187013226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T21:15:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:53,363 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f26c528f6f4d434d96906e005398b834 2024-11-09T21:15:53,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/14114e03c1ed45e8a5ad3f06546a833a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/14114e03c1ed45e8a5ad3f06546a833a 2024-11-09T21:15:53,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/14114e03c1ed45e8a5ad3f06546a833a, entries=200, sequenceid=277, filesize=14.4 K 2024-11-09T21:15:53,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/f6c36c6938184c489c68d169e9be2d59 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f6c36c6938184c489c68d169e9be2d59 2024-11-09T21:15:53,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f6c36c6938184c489c68d169e9be2d59, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T21:15:53,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f26c528f6f4d434d96906e005398b834 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f26c528f6f4d434d96906e005398b834 2024-11-09T21:15:53,412 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f26c528f6f4d434d96906e005398b834, entries=150, sequenceid=277, filesize=12.0 K 2024-11-09T21:15:53,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for ac01e9e325a29ffdc19c65c95fe0a1df in 934ms, sequenceid=277, compaction requested=true 2024-11-09T21:15:53,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:53,414 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:53,414 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:53,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:53,416 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:53,416 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:53,416 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,417 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/73fa3721bd80485c8fb7e5d9f8936371, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/aacec4063b824f12a90da30fbafd90de, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f6c36c6938184c489c68d169e9be2d59] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.4 K 2024-11-09T21:15:53,417 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39671 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:53,417 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:53,417 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,418 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d6987dc8d6f43828825af97988d27ea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/fafcb343dd3a42d9a14823cf2b1aca7c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/14114e03c1ed45e8a5ad3f06546a833a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=38.7 K 2024-11-09T21:15:53,418 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 73fa3721bd80485c8fb7e5d9f8936371, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731186950968 2024-11-09T21:15:53,418 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting aacec4063b824f12a90da30fbafd90de, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731186951164 2024-11-09T21:15:53,419 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d6987dc8d6f43828825af97988d27ea, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731186950968 2024-11-09T21:15:53,419 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting fafcb343dd3a42d9a14823cf2b1aca7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731186951164 2024-11-09T21:15:53,419 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f6c36c6938184c489c68d169e9be2d59, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731186951836 2024-11-09T21:15:53,420 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14114e03c1ed45e8a5ad3f06546a833a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731186951829 2024-11-09T21:15:53,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-09T21:15:53,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,445 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=120.76 KB heapSize=335.77 KB 2024-11-09T21:15:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:53,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:53,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:53,453 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#51 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:53,454 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/76fd4dba43c5480bbacdb0324d24d9c7 is 50, key is test_row_0/B:col10/1731186952478/Put/seqid=0 2024-11-09T21:15:53,455 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:53,455 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/f2ff97e31da64fd684dc57edb9ee6230 is 50, key is test_row_0/A:col10/1731186952478/Put/seqid=0 2024-11-09T21:15:53,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1ca472dc3c6244e2bf4edc297179a990 is 50, key is test_row_0/A:col10/1731186952570/Put/seqid=0 2024-11-09T21:15:53,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741888_1064 (size=12881) 2024-11-09T21:15:53,505 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/76fd4dba43c5480bbacdb0324d24d9c7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/76fd4dba43c5480bbacdb0324d24d9c7 2024-11-09T21:15:53,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741890_1066 (size=12881) 2024-11-09T21:15:53,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741889_1065 (size=12301) 2024-11-09T21:15:53,520 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/f2ff97e31da64fd684dc57edb9ee6230 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/f2ff97e31da64fd684dc57edb9ee6230 2024-11-09T21:15:53,521 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1ca472dc3c6244e2bf4edc297179a990 2024-11-09T21:15:53,535 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into 76fd4dba43c5480bbacdb0324d24d9c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:53,535 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:53,535 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=13, startTime=1731186953414; duration=0sec 2024-11-09T21:15:53,536 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:53,536 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:53,536 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:53,541 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:53,541 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:53,541 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into f2ff97e31da64fd684dc57edb9ee6230(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:53,541 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,541 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:53,541 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=13, startTime=1731186953413; duration=0sec 2024-11-09T21:15:53,541 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/476a900f31564508920c484fdd26eb25, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ed94f44ea04347cab1366b63b4f0742b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f26c528f6f4d434d96906e005398b834] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.4 K 2024-11-09T21:15:53,541 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:53,541 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:53,542 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 476a900f31564508920c484fdd26eb25, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731186950968 2024-11-09T21:15:53,545 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ed94f44ea04347cab1366b63b4f0742b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=262, earliestPutTs=1731186951164 2024-11-09T21:15:53,545 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f26c528f6f4d434d96906e005398b834, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731186951836 2024-11-09T21:15:53,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cbf037b8c084473aa33b0ce2e6097190 is 50, key is test_row_0/B:col10/1731186952570/Put/seqid=0 2024-11-09T21:15:53,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741891_1067 (size=12301) 2024-11-09T21:15:53,589 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cbf037b8c084473aa33b0ce2e6097190 2024-11-09T21:15:53,607 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#55 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:53,607 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/abd16081fea14b0c9d7564d8d3382a9f is 50, key is test_row_0/C:col10/1731186952478/Put/seqid=0 2024-11-09T21:15:53,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/76ae0d25114e47b485f80a1c572a2ab1 is 50, key is test_row_0/C:col10/1731186952570/Put/seqid=0 2024-11-09T21:15:53,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741893_1069 (size=12301) 2024-11-09T21:15:53,634 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/76ae0d25114e47b485f80a1c572a2ab1 2024-11-09T21:15:53,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T21:15:53,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741892_1068 (size=12881) 2024-11-09T21:15:53,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/1ca472dc3c6244e2bf4edc297179a990 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1ca472dc3c6244e2bf4edc297179a990 2024-11-09T21:15:53,654 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/abd16081fea14b0c9d7564d8d3382a9f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/abd16081fea14b0c9d7564d8d3382a9f 2024-11-09T21:15:53,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1ca472dc3c6244e2bf4edc297179a990, entries=150, sequenceid=310, filesize=12.0 K 2024-11-09T21:15:53,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/cbf037b8c084473aa33b0ce2e6097190 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cbf037b8c084473aa33b0ce2e6097190 2024-11-09T21:15:53,695 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into abd16081fea14b0c9d7564d8d3382a9f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:53,695 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:53,695 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=13, startTime=1731186953414; duration=0sec 2024-11-09T21:15:53,695 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:53,695 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:53,698 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cbf037b8c084473aa33b0ce2e6097190, entries=150, sequenceid=310, filesize=12.0 K 2024-11-09T21:15:53,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:53,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:53,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/76ae0d25114e47b485f80a1c572a2ab1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/76ae0d25114e47b485f80a1c572a2ab1 2024-11-09T21:15:53,709 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/76ae0d25114e47b485f80a1c572a2ab1, entries=150, sequenceid=310, filesize=12.0 K 2024-11-09T21:15:53,710 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~335.72 KB/343776, currentSize=20.13 KB/20610 for ac01e9e325a29ffdc19c65c95fe0a1df in 265ms, sequenceid=310, compaction requested=false 2024-11-09T21:15:53,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:53,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-09T21:15:53,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-09T21:15:53,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-09T21:15:53,718 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2120 sec 2024-11-09T21:15:53,722 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:15:53,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:53,724 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:53,725 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:53,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.2230 sec 2024-11-09T21:15:53,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c295c8ae498c46e8a8923c65db6c51e6 is 50, key is test_row_0/A:col10/1731186953720/Put/seqid=0 2024-11-09T21:15:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187013776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186953730, max timestamp=1731186953778Num uniques 859; 2024-11-09T21:15:53,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186953730, max timestamp=1731186953778Num uniques 859; 2024-11-09T21:15:53,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:53,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:53,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186953730, max timestamp=1731186953778Num uniques 859; 2024-11-09T21:15:53,782 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187013771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741894_1070 (size=12301) 2024-11-09T21:15:53,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c295c8ae498c46e8a8923c65db6c51e6 2024-11-09T21:15:53,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/66b9b971423948fca2a5062e7065b1e3 is 50, key is test_row_0/B:col10/1731186953720/Put/seqid=0 2024-11-09T21:15:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,860 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187013858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187013851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,861 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741895_1071 (size=12301) 2024-11-09T21:15:53,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/66b9b971423948fca2a5062e7065b1e3 2024-11-09T21:15:53,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187013860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,869 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f8cc069a6de2447ea8233903e2e8fc64 is 50, key is test_row_0/C:col10/1731186953720/Put/seqid=0 2024-11-09T21:15:53,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187013886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741896_1072 (size=12301) 2024-11-09T21:15:53,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f8cc069a6de2447ea8233903e2e8fc64 2024-11-09T21:15:53,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:53,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187013897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:53,914 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:53,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c295c8ae498c46e8a8923c65db6c51e6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c295c8ae498c46e8a8923c65db6c51e6 2024-11-09T21:15:53,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c295c8ae498c46e8a8923c65db6c51e6, entries=150, sequenceid=324, filesize=12.0 K 2024-11-09T21:15:53,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/66b9b971423948fca2a5062e7065b1e3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/66b9b971423948fca2a5062e7065b1e3 2024-11-09T21:15:53,935 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/66b9b971423948fca2a5062e7065b1e3, entries=150, sequenceid=324, filesize=12.0 K 2024-11-09T21:15:53,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f8cc069a6de2447ea8233903e2e8fc64 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f8cc069a6de2447ea8233903e2e8fc64 2024-11-09T21:15:53,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f8cc069a6de2447ea8233903e2e8fc64, entries=150, sequenceid=324, filesize=12.0 K 2024-11-09T21:15:53,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ac01e9e325a29ffdc19c65c95fe0a1df in 230ms, sequenceid=324, compaction requested=true 2024-11-09T21:15:53,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:53,952 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:53,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:53,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:53,952 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:53,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:53,954 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:53,954 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:53,955 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,955 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/f2ff97e31da64fd684dc57edb9ee6230, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1ca472dc3c6244e2bf4edc297179a990, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c295c8ae498c46e8a8923c65db6c51e6] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.6 K 2024-11-09T21:15:53,957 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2ff97e31da64fd684dc57edb9ee6230, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731186951836 2024-11-09T21:15:53,959 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:53,960 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:53,960 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:53,960 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/76fd4dba43c5480bbacdb0324d24d9c7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cbf037b8c084473aa33b0ce2e6097190, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/66b9b971423948fca2a5062e7065b1e3] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.6 K 2024-11-09T21:15:53,964 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca472dc3c6244e2bf4edc297179a990, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731186952567 2024-11-09T21:15:53,964 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 76fd4dba43c5480bbacdb0324d24d9c7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731186951836 2024-11-09T21:15:53,964 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c295c8ae498c46e8a8923c65db6c51e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731186953697 2024-11-09T21:15:53,964 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting cbf037b8c084473aa33b0ce2e6097190, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731186952567 2024-11-09T21:15:53,965 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 66b9b971423948fca2a5062e7065b1e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731186953697 2024-11-09T21:15:53,986 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731186953837, max timestamp=1731186953981Num uniques 1018; 2024-11-09T21:15:53,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731186953837, max timestamp=1731186953981Num uniques 1018; 2024-11-09T21:15:53,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:15:53,987 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:15:53,988 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=64.85 KB heapSize=178.80 KB 2024-11-09T21:15:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:53,989 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:15:53,998 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:53,999 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/0b10e03257344d5d97964f9acdceab15 is 50, key is test_row_0/B:col10/1731186953720/Put/seqid=0 2024-11-09T21:15:54,001 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#60 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:54,002 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/147e532c66db4264ab30f8a780757360 is 50, key is test_row_0/A:col10/1731186953720/Put/seqid=0 2024-11-09T21:15:54,022 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b731b5e6e24c49d986e5ed0164bfcdfd is 50, key is test_row_0/A:col10/1731186953987/Put/seqid=0 2024-11-09T21:15:54,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741897_1073 (size=12983) 2024-11-09T21:15:54,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741899_1075 (size=17181) 2024-11-09T21:15:54,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187014040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,050 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187014047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187014050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741898_1074 (size=12983) 2024-11-09T21:15:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187014096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,102 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/147e532c66db4264ab30f8a780757360 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/147e532c66db4264ab30f8a780757360 2024-11-09T21:15:54,102 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/0b10e03257344d5d97964f9acdceab15 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/0b10e03257344d5d97964f9acdceab15 2024-11-09T21:15:54,114 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into 0b10e03257344d5d97964f9acdceab15(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:54,114 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:54,114 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=13, startTime=1731186953952; duration=0sec 2024-11-09T21:15:54,114 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:54,115 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:54,115 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:54,115 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into 147e532c66db4264ab30f8a780757360(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:54,115 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:54,115 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=13, startTime=1731186953952; duration=0sec 2024-11-09T21:15:54,116 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:54,116 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:54,117 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:54,117 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:54,117 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:54,117 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/abd16081fea14b0c9d7564d8d3382a9f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/76ae0d25114e47b485f80a1c572a2ab1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f8cc069a6de2447ea8233903e2e8fc64] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.6 K 2024-11-09T21:15:54,118 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting abd16081fea14b0c9d7564d8d3382a9f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731186951836 2024-11-09T21:15:54,119 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 76ae0d25114e47b485f80a1c572a2ab1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731186952567 2024-11-09T21:15:54,120 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f8cc069a6de2447ea8233903e2e8fc64, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731186953697 2024-11-09T21:15:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187014127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,135 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#63 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:54,135 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e53478e3578f49d89f7899eb9974cbab is 50, key is test_row_0/C:col10/1731186953720/Put/seqid=0 2024-11-09T21:15:54,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741900_1076 (size=12983) 2024-11-09T21:15:54,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187014156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187014156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,163 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,170 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e53478e3578f49d89f7899eb9974cbab as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e53478e3578f49d89f7899eb9974cbab 2024-11-09T21:15:54,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187014167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,182 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into e53478e3578f49d89f7899eb9974cbab(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:54,182 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:54,182 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=13, startTime=1731186953953; duration=0sec 2024-11-09T21:15:54,182 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:54,182 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:54,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187014366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187014366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,372 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,373 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,383 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187014377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,384 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187014408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187014437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b731b5e6e24c49d986e5ed0164bfcdfd 2024-11-09T21:15:54,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/9d9b520508bc4b6aa24d35e86454f6cd is 50, key is test_row_0/B:col10/1731186953987/Put/seqid=0 2024-11-09T21:15:54,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741901_1077 (size=12301) 2024-11-09T21:15:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-09T21:15:54,657 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:54,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-09T21:15:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:15:54,661 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:54,662 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:54,663 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:54,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187014676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187014676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,683 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,684 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187014686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:15:54,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T21:15:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:54,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:54,817 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:54,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:54,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/9d9b520508bc4b6aa24d35e86454f6cd 2024-11-09T21:15:54,907 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/bc8edd93fdf3485696909eb7fdfc8c74 is 50, key is test_row_0/C:col10/1731186953987/Put/seqid=0 2024-11-09T21:15:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187014916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741902_1078 (size=12301) 2024-11-09T21:15:54,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:54,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187014947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:54,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/bc8edd93fdf3485696909eb7fdfc8c74 2024-11-09T21:15:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:54,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:54,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b731b5e6e24c49d986e5ed0164bfcdfd as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b731b5e6e24c49d986e5ed0164bfcdfd 2024-11-09T21:15:54,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T21:15:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:54,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:54,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:54,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:54,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:54,976 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b731b5e6e24c49d986e5ed0164bfcdfd, entries=250, sequenceid=363, filesize=16.8 K 2024-11-09T21:15:54,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:15:54,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/9d9b520508bc4b6aa24d35e86454f6cd as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/9d9b520508bc4b6aa24d35e86454f6cd 2024-11-09T21:15:54,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/9d9b520508bc4b6aa24d35e86454f6cd, entries=150, sequenceid=363, filesize=12.0 K 2024-11-09T21:15:54,993 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/bc8edd93fdf3485696909eb7fdfc8c74 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bc8edd93fdf3485696909eb7fdfc8c74 2024-11-09T21:15:55,016 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bc8edd93fdf3485696909eb7fdfc8c74, entries=150, sequenceid=363, filesize=12.0 K 2024-11-09T21:15:55,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~71.56 KB/73280, heapSize ~198.67 KB/203440, currentSize=107.34 KB/109920 for ac01e9e325a29ffdc19c65c95fe0a1df in 1030ms, sequenceid=363, compaction requested=false 2024-11-09T21:15:55,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:55,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-09T21:15:55,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:55,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T21:15:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:55,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:55,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/efbabfb1da59490397503b6acb572cea is 50, key is test_row_0/A:col10/1731186954022/Put/seqid=0 2024-11-09T21:15:55,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:55,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:55,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741903_1079 (size=12301) 2024-11-09T21:15:55,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187015230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,237 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,238 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187015235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187015236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:15:55,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187015349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187015349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,358 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,363 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187015356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,572 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187015567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187015567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187015576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,581 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/efbabfb1da59490397503b6acb572cea 2024-11-09T21:15:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/bfe6a44787214f0f9aec9a9a8fc91e54 is 50, key is test_row_0/B:col10/1731186954022/Put/seqid=0 2024-11-09T21:15:55,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741904_1080 (size=12301) 2024-11-09T21:15:55,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:15:55,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187015877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,880 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187015886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187015886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,932 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187015926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,938 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in ac01e9e325a29ffdc19c65c95fe0a1df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:15:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:55,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:55,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187015966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:55,969 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in ac01e9e325a29ffdc19c65c95fe0a1df of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:15:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:55,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,008 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/bfe6a44787214f0f9aec9a9a8fc91e54 2024-11-09T21:15:56,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e642d9833c814033a62fb7a776679de4 is 50, key is test_row_0/C:col10/1731186954022/Put/seqid=0 2024-11-09T21:15:56,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741905_1081 (size=12301) 2024-11-09T21:15:56,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e642d9833c814033a62fb7a776679de4 2024-11-09T21:15:56,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/efbabfb1da59490397503b6acb572cea as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/efbabfb1da59490397503b6acb572cea 2024-11-09T21:15:56,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/efbabfb1da59490397503b6acb572cea, entries=150, sequenceid=385, filesize=12.0 K 2024-11-09T21:15:56,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/bfe6a44787214f0f9aec9a9a8fc91e54 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/bfe6a44787214f0f9aec9a9a8fc91e54 2024-11-09T21:15:56,067 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/bfe6a44787214f0f9aec9a9a8fc91e54, entries=150, sequenceid=385, filesize=12.0 K 2024-11-09T21:15:56,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/e642d9833c814033a62fb7a776679de4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e642d9833c814033a62fb7a776679de4 2024-11-09T21:15:56,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e642d9833c814033a62fb7a776679de4, entries=150, sequenceid=385, filesize=12.0 K 2024-11-09T21:15:56,080 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for ac01e9e325a29ffdc19c65c95fe0a1df in 953ms, sequenceid=385, compaction requested=true 2024-11-09T21:15:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-09T21:15:56,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-09T21:15:56,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-09T21:15:56,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4200 sec 2024-11-09T21:15:56,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.4280 sec 2024-11-09T21:15:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:56,389 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T21:15:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:56,389 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:56,406 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/53255a70bc094466b7edb55b38d05c6f is 50, key is test_row_0/A:col10/1731186956387/Put/seqid=0 2024-11-09T21:15:56,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187016433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,444 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,444 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187016434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187016433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741906_1082 (size=14741) 2024-11-09T21:15:56,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187016546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,552 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187016556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187016556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187016756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,766 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187016766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:56,770 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187016767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:56,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-09T21:15:56,816 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:56,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-09T21:15:56,819 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:56,820 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:56,820 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T21:15:56,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/53255a70bc094466b7edb55b38d05c6f 2024-11-09T21:15:56,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/baf11684c6e44afba9aa833f763ebff4 is 50, key is test_row_0/B:col10/1731186956387/Put/seqid=0 2024-11-09T21:15:56,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741907_1083 (size=12301) 2024-11-09T21:15:56,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T21:15:56,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T21:15:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:56,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:56,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:57,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187017076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:57,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187017076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:57,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187017076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:57,085 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:57,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T21:15:57,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T21:15:57,275 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/baf11684c6e44afba9aa833f763ebff4 2024-11-09T21:15:57,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T21:15:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:57,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,284 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:57,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/eb68a02aef734bdeba97c7839f6c12c6 is 50, key is test_row_0/C:col10/1731186956387/Put/seqid=0 2024-11-09T21:15:57,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741908_1084 (size=12301) 2024-11-09T21:15:57,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=399 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/eb68a02aef734bdeba97c7839f6c12c6 2024-11-09T21:15:57,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/53255a70bc094466b7edb55b38d05c6f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/53255a70bc094466b7edb55b38d05c6f 2024-11-09T21:15:57,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/53255a70bc094466b7edb55b38d05c6f, entries=200, sequenceid=399, filesize=14.4 K 2024-11-09T21:15:57,344 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/baf11684c6e44afba9aa833f763ebff4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/baf11684c6e44afba9aa833f763ebff4 2024-11-09T21:15:57,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/baf11684c6e44afba9aa833f763ebff4, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T21:15:57,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/eb68a02aef734bdeba97c7839f6c12c6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/eb68a02aef734bdeba97c7839f6c12c6 2024-11-09T21:15:57,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/eb68a02aef734bdeba97c7839f6c12c6, entries=150, sequenceid=399, filesize=12.0 K 2024-11-09T21:15:57,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for ac01e9e325a29ffdc19c65c95fe0a1df in 989ms, sequenceid=399, compaction requested=true 2024-11-09T21:15:57,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:57,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:57,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:57,378 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:57,378 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:57,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:57,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:57,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:57,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:57,382 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57206 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:57,382 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:57,382 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,382 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/147e532c66db4264ab30f8a780757360, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b731b5e6e24c49d986e5ed0164bfcdfd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/efbabfb1da59490397503b6acb572cea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/53255a70bc094466b7edb55b38d05c6f] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=55.9 K 2024-11-09T21:15:57,384 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:57,384 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:57,384 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,385 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/0b10e03257344d5d97964f9acdceab15, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/9d9b520508bc4b6aa24d35e86454f6cd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/bfe6a44787214f0f9aec9a9a8fc91e54, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/baf11684c6e44afba9aa833f763ebff4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=48.7 K 2024-11-09T21:15:57,385 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 147e532c66db4264ab30f8a780757360, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731186953697 2024-11-09T21:15:57,385 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b10e03257344d5d97964f9acdceab15, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731186953697 2024-11-09T21:15:57,385 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b731b5e6e24c49d986e5ed0164bfcdfd, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731186953837 2024-11-09T21:15:57,386 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d9b520508bc4b6aa24d35e86454f6cd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731186953855 2024-11-09T21:15:57,386 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting efbabfb1da59490397503b6acb572cea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731186954022 2024-11-09T21:15:57,386 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bfe6a44787214f0f9aec9a9a8fc91e54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731186954022 2024-11-09T21:15:57,386 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53255a70bc094466b7edb55b38d05c6f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731186955213 2024-11-09T21:15:57,386 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting baf11684c6e44afba9aa833f763ebff4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731186955230 2024-11-09T21:15:57,407 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#72 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:57,408 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/69a9ebc9ac9b4306b1a7fef3d066b91c is 50, key is test_row_0/B:col10/1731186956387/Put/seqid=0 2024-11-09T21:15:57,411 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:57,412 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b7106b10aa234bc5806b17cbb5d8a200 is 50, key is test_row_0/A:col10/1731186956387/Put/seqid=0 2024-11-09T21:15:57,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741909_1085 (size=13119) 2024-11-09T21:15:57,437 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/69a9ebc9ac9b4306b1a7fef3d066b91c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/69a9ebc9ac9b4306b1a7fef3d066b91c 2024-11-09T21:15:57,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,438 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:57,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T21:15:57,447 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into 69a9ebc9ac9b4306b1a7fef3d066b91c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:57,448 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:57,448 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=12, startTime=1731186957378; duration=0sec 2024-11-09T21:15:57,448 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:57,448 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:57,448 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:57,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741910_1086 (size=13119) 2024-11-09T21:15:57,451 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:57,451 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:57,452 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,452 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e53478e3578f49d89f7899eb9974cbab, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bc8edd93fdf3485696909eb7fdfc8c74, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e642d9833c814033a62fb7a776679de4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/eb68a02aef734bdeba97c7839f6c12c6] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=48.7 K 2024-11-09T21:15:57,458 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e53478e3578f49d89f7899eb9974cbab, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731186953697 2024-11-09T21:15:57,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b1241914fbf34027a28b200172facdf6 is 50, key is test_row_0/A:col10/1731186956429/Put/seqid=0 2024-11-09T21:15:57,460 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bc8edd93fdf3485696909eb7fdfc8c74, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731186953855 2024-11-09T21:15:57,460 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b7106b10aa234bc5806b17cbb5d8a200 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b7106b10aa234bc5806b17cbb5d8a200 2024-11-09T21:15:57,464 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e642d9833c814033a62fb7a776679de4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731186954022 2024-11-09T21:15:57,465 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting eb68a02aef734bdeba97c7839f6c12c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731186955230 2024-11-09T21:15:57,470 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into b7106b10aa234bc5806b17cbb5d8a200(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:57,470 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:57,470 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=12, startTime=1731186957378; duration=0sec 2024-11-09T21:15:57,472 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:57,472 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:57,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741911_1087 (size=12301) 2024-11-09T21:15:57,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b1241914fbf34027a28b200172facdf6 2024-11-09T21:15:57,492 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#75 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:57,493 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/cfa9d5e3c8144b7a982b0e423797329f is 50, key is test_row_0/C:col10/1731186956387/Put/seqid=0 2024-11-09T21:15:57,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/f03c9ee236d0436caea7869340db6d7a is 50, key is test_row_0/B:col10/1731186956429/Put/seqid=0 2024-11-09T21:15:57,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741912_1088 (size=13119) 2024-11-09T21:15:57,513 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/cfa9d5e3c8144b7a982b0e423797329f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/cfa9d5e3c8144b7a982b0e423797329f 2024-11-09T21:15:57,520 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into cfa9d5e3c8144b7a982b0e423797329f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:57,520 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:57,520 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=12, startTime=1731186957378; duration=0sec 2024-11-09T21:15:57,520 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:57,520 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:57,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741913_1089 (size=12301) 2024-11-09T21:15:57,524 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/f03c9ee236d0436caea7869340db6d7a 2024-11-09T21:15:57,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f85dc07644d047458e3dc1bb8784c609 is 50, key is test_row_0/C:col10/1731186956429/Put/seqid=0 2024-11-09T21:15:57,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741914_1090 (size=12301) 2024-11-09T21:15:57,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f85dc07644d047458e3dc1bb8784c609 2024-11-09T21:15:57,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/b1241914fbf34027a28b200172facdf6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b1241914fbf34027a28b200172facdf6 2024-11-09T21:15:57,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b1241914fbf34027a28b200172facdf6, entries=150, sequenceid=417, filesize=12.0 K 2024-11-09T21:15:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/f03c9ee236d0436caea7869340db6d7a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f03c9ee236d0436caea7869340db6d7a 2024-11-09T21:15:57,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:57,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:57,614 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f03c9ee236d0436caea7869340db6d7a, entries=150, sequenceid=417, filesize=12.0 K 2024-11-09T21:15:57,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f85dc07644d047458e3dc1bb8784c609 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f85dc07644d047458e3dc1bb8784c609 2024-11-09T21:15:57,623 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f85dc07644d047458e3dc1bb8784c609, entries=150, sequenceid=417, filesize=12.0 K 2024-11-09T21:15:57,625 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=53.67 KB/54960 for ac01e9e325a29ffdc19c65c95fe0a1df in 186ms, sequenceid=417, compaction requested=false 2024-11-09T21:15:57,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:57,625 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:15:57,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:57,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:57,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:57,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:57,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:57,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:57,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:57,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-09T21:15:57,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-09T21:15:57,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-09T21:15:57,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 810 msec 2024-11-09T21:15:57,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 819 msec 2024-11-09T21:15:57,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/e81fbc571285455394687d5922f4ff64 is 50, key is test_row_0/A:col10/1731186957624/Put/seqid=0 2024-11-09T21:15:57,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741915_1091 (size=22065) 2024-11-09T21:15:57,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/e81fbc571285455394687d5922f4ff64 2024-11-09T21:15:57,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/ea570b1f6dec4210b5d51fd0f4f894a9 is 50, key is test_row_0/B:col10/1731186957624/Put/seqid=0 2024-11-09T21:15:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187017676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187017677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:57,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:57,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187017683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:57,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:57,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741916_1092 (size=12301) 2024-11-09T21:15:57,696 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/ea570b1f6dec4210b5d51fd0f4f894a9 2024-11-09T21:15:57,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/bed554bdaaf7448f802d70ae996068fb is 50, key is test_row_0/C:col10/1731186957624/Put/seqid=0 2024-11-09T21:15:57,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741917_1093 (size=12301) 2024-11-09T21:15:57,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=433 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/bed554bdaaf7448f802d70ae996068fb 2024-11-09T21:15:57,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/e81fbc571285455394687d5922f4ff64 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e81fbc571285455394687d5922f4ff64 2024-11-09T21:15:57,747 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e81fbc571285455394687d5922f4ff64, entries=350, sequenceid=433, filesize=21.5 K 2024-11-09T21:15:57,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/ea570b1f6dec4210b5d51fd0f4f894a9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ea570b1f6dec4210b5d51fd0f4f894a9 2024-11-09T21:15:57,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ea570b1f6dec4210b5d51fd0f4f894a9, entries=150, sequenceid=433, filesize=12.0 K 2024-11-09T21:15:57,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/bed554bdaaf7448f802d70ae996068fb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bed554bdaaf7448f802d70ae996068fb 2024-11-09T21:15:57,777 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bed554bdaaf7448f802d70ae996068fb, entries=150, sequenceid=433, filesize=12.0 K 2024-11-09T21:15:57,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for ac01e9e325a29ffdc19c65c95fe0a1df in 162ms, sequenceid=433, compaction requested=true 2024-11-09T21:15:57,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:57,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186957661, max timestamp=1731186957789Num uniques 859; 2024-11-09T21:15:57,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186957661, max timestamp=1731186957789Num uniques 859; 2024-11-09T21:15:57,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:57,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:57,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186957661, max timestamp=1731186957789Num uniques 859; 2024-11-09T21:15:57,794 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:57,794 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:57,796 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47485 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:57,796 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:57,796 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,796 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b7106b10aa234bc5806b17cbb5d8a200, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b1241914fbf34027a28b200172facdf6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e81fbc571285455394687d5922f4ff64] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=46.4 K 2024-11-09T21:15:57,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:57,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:57,797 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:57,797 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7106b10aa234bc5806b17cbb5d8a200, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731186955230 2024-11-09T21:15:57,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:57,797 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1241914fbf34027a28b200172facdf6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731186956429 2024-11-09T21:15:57,797 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:57,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:57,798 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:57,798 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:57,798 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:57,798 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:57,803 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/69a9ebc9ac9b4306b1a7fef3d066b91c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f03c9ee236d0436caea7869340db6d7a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ea570b1f6dec4210b5d51fd0f4f894a9] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.8 K 2024-11-09T21:15:57,798 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e81fbc571285455394687d5922f4ff64, keycount=350, bloomtype=ROW, size=21.5 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731186957602 2024-11-09T21:15:57,806 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 69a9ebc9ac9b4306b1a7fef3d066b91c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731186955230 2024-11-09T21:15:57,798 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=33.54 KB heapSize=81.47 KB 2024-11-09T21:15:57,809 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f03c9ee236d0436caea7869340db6d7a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731186956429 2024-11-09T21:15:57,809 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ea570b1f6dec4210b5d51fd0f4f894a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731186957623 2024-11-09T21:15:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:57,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:57,855 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#81 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:57,856 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/409248b18c484833aebf3498fdac6cf7 is 50, key is test_row_0/A:col10/1731186957624/Put/seqid=0 2024-11-09T21:15:57,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d8e67afd3bb7416ead52f3901a1746a4 is 50, key is test_row_0/A:col10/1731186957804/Put/seqid=0 2024-11-09T21:15:57,864 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#83 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:57,865 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/d3e30f8819484369b1efd8623a0c107d is 50, key is test_row_0/B:col10/1731186957624/Put/seqid=0 2024-11-09T21:15:57,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741918_1094 (size=13221) 2024-11-09T21:15:57,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741919_1095 (size=12301) 2024-11-09T21:15:57,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741920_1096 (size=13221) 2024-11-09T21:15:57,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-09T21:15:57,958 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:15:57,958 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:15:57,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-09T21:15:57,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T21:15:57,960 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:15:57,961 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:15:57,961 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:15:57,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186957926, max timestamp=1731186957969Num uniques 859; 2024-11-09T21:15:57,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186957926, max timestamp=1731186957969Num uniques 859; 2024-11-09T21:15:57,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:57,973 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:57,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186957926, max timestamp=1731186957969Num uniques 859; 2024-11-09T21:15:57,974 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:58,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187018003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187018004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187018005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187018008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187018008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,013 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T21:15:58,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T21:15:58,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:58,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187018116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,122 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,122 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187018117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187018117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187018117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187018117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,124 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T21:15:58,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d8e67afd3bb7416ead52f3901a1746a4 2024-11-09T21:15:58,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T21:15:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,296 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/d3e30f8819484369b1efd8623a0c107d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/d3e30f8819484369b1efd8623a0c107d 2024-11-09T21:15:58,301 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/409248b18c484833aebf3498fdac6cf7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/409248b18c484833aebf3498fdac6cf7 2024-11-09T21:15:58,309 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into d3e30f8819484369b1efd8623a0c107d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:58,309 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:58,309 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=13, startTime=1731186957797; duration=0sec 2024-11-09T21:15:58,309 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:58,309 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:58,310 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:15:58,316 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:15:58,316 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:58,316 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,316 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/cfa9d5e3c8144b7a982b0e423797329f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f85dc07644d047458e3dc1bb8784c609, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bed554bdaaf7448f802d70ae996068fb] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=36.8 K 2024-11-09T21:15:58,317 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting cfa9d5e3c8144b7a982b0e423797329f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=399, earliestPutTs=1731186955230 2024-11-09T21:15:58,318 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f85dc07644d047458e3dc1bb8784c609, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=417, earliestPutTs=1731186956429 2024-11-09T21:15:58,318 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bed554bdaaf7448f802d70ae996068fb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731186957623 2024-11-09T21:15:58,323 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into 409248b18c484833aebf3498fdac6cf7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:58,323 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:58,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/fd7d9b2e49764a4583cb00b24d6fef7f is 50, key is test_row_0/B:col10/1731186957804/Put/seqid=0 2024-11-09T21:15:58,323 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=13, startTime=1731186957794; duration=0sec 2024-11-09T21:15:58,325 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:58,325 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:58,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187018327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,342 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#85 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:58,343 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/1640102f2da9460aba36b2d19e33dd64 is 50, key is test_row_0/C:col10/1731186957624/Put/seqid=0 2024-11-09T21:15:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,343 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187018337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,345 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741921_1097 (size=12301) 2024-11-09T21:15:58,356 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/fd7d9b2e49764a4583cb00b24d6fef7f 2024-11-09T21:15:58,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/4a02f0505c4748c0b917e9e27aae04c9 is 50, key is test_row_0/C:col10/1731186957804/Put/seqid=0 2024-11-09T21:15:58,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741922_1098 (size=13221) 2024-11-09T21:15:58,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741923_1099 (size=12301) 2024-11-09T21:15:58,398 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/1640102f2da9460aba36b2d19e33dd64 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1640102f2da9460aba36b2d19e33dd64 2024-11-09T21:15:58,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/4a02f0505c4748c0b917e9e27aae04c9 2024-11-09T21:15:58,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/d8e67afd3bb7416ead52f3901a1746a4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d8e67afd3bb7416ead52f3901a1746a4 2024-11-09T21:15:58,415 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into 1640102f2da9460aba36b2d19e33dd64(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:58,415 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:58,415 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=13, startTime=1731186957798; duration=0sec 2024-11-09T21:15:58,415 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:58,416 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:15:58,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d8e67afd3bb7416ead52f3901a1746a4, entries=150, sequenceid=457, filesize=12.0 K 2024-11-09T21:15:58,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/fd7d9b2e49764a4583cb00b24d6fef7f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/fd7d9b2e49764a4583cb00b24d6fef7f 2024-11-09T21:15:58,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T21:15:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:15:58,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/fd7d9b2e49764a4583cb00b24d6fef7f, entries=150, sequenceid=457, filesize=12.0 K 2024-11-09T21:15:58,453 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/4a02f0505c4748c0b917e9e27aae04c9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/4a02f0505c4748c0b917e9e27aae04c9 2024-11-09T21:15:58,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/4a02f0505c4748c0b917e9e27aae04c9, entries=150, sequenceid=457, filesize=12.0 K 2024-11-09T21:15:58,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.19 KB/144576, currentSize=134.18 KB/137400 for ac01e9e325a29ffdc19c65c95fe0a1df in 664ms, sequenceid=457, compaction requested=false 2024-11-09T21:15:58,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:58,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T21:15:58,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-09T21:15:58,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:58,600 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-09T21:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:58,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/e2892405c3cf4b07a1b1d34607170be8 is 50, key is test_row_0/A:col10/1731186958006/Put/seqid=0 2024-11-09T21:15:58,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741924_1100 (size=12301) 2024-11-09T21:15:58,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/e2892405c3cf4b07a1b1d34607170be8 2024-11-09T21:15:58,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c5364be58db34fb1bfcfc7826245683f is 50, key is test_row_0/B:col10/1731186958006/Put/seqid=0 2024-11-09T21:15:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:58,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:15:58,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741925_1101 (size=12301) 2024-11-09T21:15:58,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c5364be58db34fb1bfcfc7826245683f 2024-11-09T21:15:58,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187018664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,674 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/0d4b91d9f20d48908a4735b438b05be8 is 50, key is test_row_0/C:col10/1731186958006/Put/seqid=0 2024-11-09T21:15:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187018666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187018667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187018671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187018673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741926_1102 (size=12301) 2024-11-09T21:15:58,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187018776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187018787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187018787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187018787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,790 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187018787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187018986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,991 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187018996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187018996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,998 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187018996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187018996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:58,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,086 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/0d4b91d9f20d48908a4735b438b05be8 2024-11-09T21:15:59,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/e2892405c3cf4b07a1b1d34607170be8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e2892405c3cf4b07a1b1d34607170be8 2024-11-09T21:15:59,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T21:15:59,102 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e2892405c3cf4b07a1b1d34607170be8, entries=150, sequenceid=496, filesize=12.0 K 2024-11-09T21:15:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/c5364be58db34fb1bfcfc7826245683f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c5364be58db34fb1bfcfc7826245683f 2024-11-09T21:15:59,110 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c5364be58db34fb1bfcfc7826245683f, entries=150, sequenceid=496, filesize=12.0 K 2024-11-09T21:15:59,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/0d4b91d9f20d48908a4735b438b05be8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/0d4b91d9f20d48908a4735b438b05be8 2024-11-09T21:15:59,116 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/0d4b91d9f20d48908a4735b438b05be8, entries=150, sequenceid=496, filesize=12.0 K 2024-11-09T21:15:59,117 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for ac01e9e325a29ffdc19c65c95fe0a1df in 517ms, sequenceid=496, compaction requested=true 2024-11-09T21:15:59,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:59,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:59,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-09T21:15:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-09T21:15:59,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-09T21:15:59,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1590 sec 2024-11-09T21:15:59,123 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1640 sec 2024-11-09T21:15:59,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:59,298 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:15:59,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:59,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:15:59,317 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/624bb8176da945e08234f933ecf0c59d is 50, key is test_row_0/A:col10/1731186959297/Put/seqid=0 2024-11-09T21:15:59,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741927_1103 (size=12301) 2024-11-09T21:15:59,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/624bb8176da945e08234f933ecf0c59d 2024-11-09T21:15:59,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/3cee9d6e1f6a44e38ccd74b02f6a5e30 is 50, key is test_row_0/B:col10/1731186959297/Put/seqid=0 2024-11-09T21:15:59,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186959349, max timestamp=1731186959359Num uniques 859; 2024-11-09T21:15:59,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:59,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186959349, max timestamp=1731186959359Num uniques 859; 2024-11-09T21:15:59,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:59,366 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186959349, max timestamp=1731186959359Num uniques 859; 2024-11-09T21:15:59,367 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:59,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741928_1104 (size=12301) 2024-11-09T21:15:59,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/3cee9d6e1f6a44e38ccd74b02f6a5e30 2024-11-09T21:15:59,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f37afba73348463eab9cda5fc01349bc is 50, key is test_row_0/C:col10/1731186959297/Put/seqid=0 2024-11-09T21:15:59,415 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187019408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,416 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187019412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741929_1105 (size=12301) 2024-11-09T21:15:59,416 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f37afba73348463eab9cda5fc01349bc 2024-11-09T21:15:59,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187019414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,422 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187019416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187019416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/624bb8176da945e08234f933ecf0c59d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/624bb8176da945e08234f933ecf0c59d 2024-11-09T21:15:59,438 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/624bb8176da945e08234f933ecf0c59d, entries=150, sequenceid=507, filesize=12.0 K 2024-11-09T21:15:59,440 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/3cee9d6e1f6a44e38ccd74b02f6a5e30 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/3cee9d6e1f6a44e38ccd74b02f6a5e30 2024-11-09T21:15:59,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/3cee9d6e1f6a44e38ccd74b02f6a5e30, entries=150, sequenceid=507, filesize=12.0 K 2024-11-09T21:15:59,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/f37afba73348463eab9cda5fc01349bc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f37afba73348463eab9cda5fc01349bc 2024-11-09T21:15:59,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f37afba73348463eab9cda5fc01349bc, entries=150, sequenceid=507, filesize=12.0 K 2024-11-09T21:15:59,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for ac01e9e325a29ffdc19c65c95fe0a1df in 179ms, sequenceid=507, compaction requested=true 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:15:59,481 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:59,481 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:15:59,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:59,484 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:59,484 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/B is initiating minor compaction (all files) 2024-11-09T21:15:59,484 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/B in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:59,484 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/d3e30f8819484369b1efd8623a0c107d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/fd7d9b2e49764a4583cb00b24d6fef7f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c5364be58db34fb1bfcfc7826245683f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/3cee9d6e1f6a44e38ccd74b02f6a5e30] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=48.9 K 2024-11-09T21:15:59,485 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:59,485 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/A is initiating minor compaction (all files) 2024-11-09T21:15:59,485 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/A in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:59,485 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/409248b18c484833aebf3498fdac6cf7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d8e67afd3bb7416ead52f3901a1746a4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e2892405c3cf4b07a1b1d34607170be8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/624bb8176da945e08234f933ecf0c59d] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=48.9 K 2024-11-09T21:15:59,485 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e30f8819484369b1efd8623a0c107d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731186957623 2024-11-09T21:15:59,485 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 409248b18c484833aebf3498fdac6cf7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731186957623 2024-11-09T21:15:59,486 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fd7d9b2e49764a4583cb00b24d6fef7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731186957676 2024-11-09T21:15:59,486 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e67afd3bb7416ead52f3901a1746a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731186957676 2024-11-09T21:15:59,486 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c5364be58db34fb1bfcfc7826245683f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731186958002 2024-11-09T21:15:59,486 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2892405c3cf4b07a1b1d34607170be8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731186958002 2024-11-09T21:15:59,486 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cee9d6e1f6a44e38ccd74b02f6a5e30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731186958669 2024-11-09T21:15:59,487 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 624bb8176da945e08234f933ecf0c59d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731186958669 2024-11-09T21:15:59,523 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#B#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:59,524 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/8adabd51abe54ede9c33184bf01eed45 is 50, key is test_row_0/B:col10/1731186959297/Put/seqid=0 2024-11-09T21:15:59,532 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-09T21:15:59,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:15:59,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731186959406, max timestamp=1731186959528Num uniques 1018; 2024-11-09T21:15:59,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:15:59,533 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731186959406, max timestamp=1731186959528Num uniques 1018; 2024-11-09T21:15:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731186959406, max timestamp=1731186959528Num uniques 1018; 2024-11-09T21:15:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:15:59,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:15:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:15:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:15:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:15:59,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:15:59,535 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#A#compaction#94 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:59,536 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/abf29577c76f4179a98f8c7095294c9c is 50, key is test_row_0/A:col10/1731186959297/Put/seqid=0 2024-11-09T21:15:59,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741930_1106 (size=13357) 2024-11-09T21:15:59,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c04ca1bdc82640b1a108d0c3397ef1ba is 50, key is test_row_0/A:col10/1731186959414/Put/seqid=0 2024-11-09T21:15:59,556 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/8adabd51abe54ede9c33184bf01eed45 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/8adabd51abe54ede9c33184bf01eed45 2024-11-09T21:15:59,565 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/B of ac01e9e325a29ffdc19c65c95fe0a1df into 8adabd51abe54ede9c33184bf01eed45(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:59,566 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:59,566 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/B, priority=12, startTime=1731186959481; duration=0sec 2024-11-09T21:15:59,566 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:15:59,566 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:15:59,566 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:15:59,568 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:15:59,568 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): ac01e9e325a29ffdc19c65c95fe0a1df/C is initiating minor compaction (all files) 2024-11-09T21:15:59,568 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of ac01e9e325a29ffdc19c65c95fe0a1df/C in TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:15:59,569 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1640102f2da9460aba36b2d19e33dd64, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/4a02f0505c4748c0b917e9e27aae04c9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/0d4b91d9f20d48908a4735b438b05be8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f37afba73348463eab9cda5fc01349bc] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp, totalSize=48.9 K 2024-11-09T21:15:59,569 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1640102f2da9460aba36b2d19e33dd64, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=433, earliestPutTs=1731186957623 2024-11-09T21:15:59,570 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a02f0505c4748c0b917e9e27aae04c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731186957676 2024-11-09T21:15:59,571 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d4b91d9f20d48908a4735b438b05be8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731186958002 2024-11-09T21:15:59,572 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f37afba73348463eab9cda5fc01349bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731186958669 2024-11-09T21:15:59,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741931_1107 (size=13357) 2024-11-09T21:15:59,589 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/abf29577c76f4179a98f8c7095294c9c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/abf29577c76f4179a98f8c7095294c9c 2024-11-09T21:15:59,593 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): ac01e9e325a29ffdc19c65c95fe0a1df#C#compaction#96 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:15:59,594 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/1e19f5f9daa341f3998a32c0aaca3fed is 50, key is test_row_0/C:col10/1731186959297/Put/seqid=0 2024-11-09T21:15:59,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741932_1108 (size=17181) 2024-11-09T21:15:59,600 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/A of ac01e9e325a29ffdc19c65c95fe0a1df into abf29577c76f4179a98f8c7095294c9c(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:15:59,600 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:15:59,600 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/A, priority=12, startTime=1731186959481; duration=0sec 2024-11-09T21:15:59,600 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:15:59,600 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:15:59,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186959580, max timestamp=1731186959595Num uniques 859; 2024-11-09T21:15:59,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731186959580, max timestamp=1731186959595Num uniques 859; 2024-11-09T21:15:59,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:59,602 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:15:59,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186959580, max timestamp=1731186959595Num uniques 859; 2024-11-09T21:15:59,606 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:15:59,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741933_1109 (size=13357) 2024-11-09T21:15:59,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731186959622, max timestamp=1731186959659Num uniques 1068; 2024-11-09T21:15:59,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-09T21:15:59,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731186959622, max timestamp=1731186959659Num uniques 1068; 2024-11-09T21:15:59,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=14558, suffix heap size=139040,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=318 2024-11-09T21:15:59,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731186959622, max timestamp=1731186959659Num uniques 1018; 2024-11-09T21:15:59,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-09T21:15:59,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187019720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187019721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187019721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187019727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187019729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187019837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187019837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187019837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187019846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:15:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,851 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187019846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:15:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:15:59,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:15:59,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c04ca1bdc82640b1a108d0c3397ef1ba 2024-11-09T21:16:00,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/1e94fe0a7eaf40edbe3907de55020d01 is 50, key is test_row_0/B:col10/1731186959414/Put/seqid=0 2024-11-09T21:16:00,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741934_1110 (size=12301) 2024-11-09T21:16:00,038 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/1e19f5f9daa341f3998a32c0aaca3fed as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1e19f5f9daa341f3998a32c0aaca3fed 2024-11-09T21:16:00,045 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in ac01e9e325a29ffdc19c65c95fe0a1df/C of ac01e9e325a29ffdc19c65c95fe0a1df into 1e19f5f9daa341f3998a32c0aaca3fed(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:00,045 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:16:00,045 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., storeName=ac01e9e325a29ffdc19c65c95fe0a1df/C, priority=12, startTime=1731186959481; duration=0sec 2024-11-09T21:16:00,045 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:00,045 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:16:00,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187020046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187020046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187020046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187020056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187020056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,056 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-09T21:16:00,117 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:00,117 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:00,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-09T21:16:00,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:16:00,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:00,120 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:00,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:00,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:16:00,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-09T21:16:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:16:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:00,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:00,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:00,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:00,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187020356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187020356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187020367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,368 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187020367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187020367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,369 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/1e94fe0a7eaf40edbe3907de55020d01 2024-11-09T21:16:00,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-09T21:16:00,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:00,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:16:00,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:00,426 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:00,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:00,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:00,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c27916042fed423fa001702605f2bca5 is 50, key is test_row_0/C:col10/1731186959414/Put/seqid=0 2024-11-09T21:16:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:16:00,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741935_1111 (size=12301) 2024-11-09T21:16:00,448 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c27916042fed423fa001702605f2bca5 2024-11-09T21:16:00,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c04ca1bdc82640b1a108d0c3397ef1ba as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c04ca1bdc82640b1a108d0c3397ef1ba 2024-11-09T21:16:00,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c04ca1bdc82640b1a108d0c3397ef1ba, entries=250, sequenceid=546, filesize=16.8 K 2024-11-09T21:16:00,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/1e94fe0a7eaf40edbe3907de55020d01 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/1e94fe0a7eaf40edbe3907de55020d01 2024-11-09T21:16:00,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/1e94fe0a7eaf40edbe3907de55020d01, entries=150, sequenceid=546, filesize=12.0 K 2024-11-09T21:16:00,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/c27916042fed423fa001702605f2bca5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c27916042fed423fa001702605f2bca5 2024-11-09T21:16:00,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c27916042fed423fa001702605f2bca5, entries=150, sequenceid=546, filesize=12.0 K 2024-11-09T21:16:00,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~85.27 KB/87312, currentSize=154.31 KB/158010 for ac01e9e325a29ffdc19c65c95fe0a1df in 950ms, sequenceid=546, compaction requested=false 2024-11-09T21:16:00,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:16:00,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-09T21:16:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:00,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-11-09T21:16:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:16:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:16:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:00,580 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:16:00,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:00,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c4a1c18f6e8d49a5be2182fb817809b5 is 50, key is test_row_0/A:col10/1731186959688/Put/seqid=0 2024-11-09T21:16:00,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741936_1112 (size=12301) 2024-11-09T21:16:00,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:16:00,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:16:00,877 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. as already flushing 2024-11-09T21:16:00,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187020885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187020887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187020889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187020889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,899 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:00,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187020890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,000 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c4a1c18f6e8d49a5be2182fb817809b5 2024-11-09T21:16:01,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187021008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187021008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187021008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187021008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187021008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,013 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/26a1a280d1474ff68bca4bf0d0625363 is 50, key is test_row_0/B:col10/1731186959688/Put/seqid=0 2024-11-09T21:16:01,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741937_1113 (size=12301) 2024-11-09T21:16:01,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187021217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187021217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187021217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187021218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187021217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:16:01,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/26a1a280d1474ff68bca4bf0d0625363 2024-11-09T21:16:01,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/68bfb89cec20429398b3130d817f0fca is 50, key is test_row_0/C:col10/1731186959688/Put/seqid=0 2024-11-09T21:16:01,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741938_1114 (size=12301) 2024-11-09T21:16:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47714 deadline: 1731187021526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47784 deadline: 1731187021526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47756 deadline: 1731187021526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47742 deadline: 1731187021526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47760 deadline: 1731187021527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:01,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:01,763 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-09T21:16:01,763 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:01,764 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-09T21:16:01,764 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:01,768 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-09T21:16:01,768 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:01,769 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,769 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,769 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:01,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:01,769 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:01,769 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,769 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,770 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,773 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-09T21:16:01,773 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:01,773 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,773 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:01,773 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:01,856 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=603 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/68bfb89cec20429398b3130d817f0fca 2024-11-09T21:16:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/c4a1c18f6e8d49a5be2182fb817809b5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c4a1c18f6e8d49a5be2182fb817809b5 2024-11-09T21:16:01,867 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c4a1c18f6e8d49a5be2182fb817809b5, entries=150, sequenceid=603, filesize=12.0 K 2024-11-09T21:16:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/26a1a280d1474ff68bca4bf0d0625363 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/26a1a280d1474ff68bca4bf0d0625363 2024-11-09T21:16:01,873 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/26a1a280d1474ff68bca4bf0d0625363, entries=150, sequenceid=603, filesize=12.0 K 2024-11-09T21:16:01,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/68bfb89cec20429398b3130d817f0fca as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/68bfb89cec20429398b3130d817f0fca 2024-11-09T21:16:01,878 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/68bfb89cec20429398b3130d817f0fca, entries=150, sequenceid=603, filesize=12.0 K 2024-11-09T21:16:01,879 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=46.96 KB/48090 for ac01e9e325a29ffdc19c65c95fe0a1df in 1299ms, sequenceid=603, compaction requested=true 2024-11-09T21:16:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:16:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:01,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-09T21:16:01,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-09T21:16:01,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-09T21:16:01,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7610 sec 2024-11-09T21:16:01,887 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7680 sec 2024-11-09T21:16:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:16:02,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing ac01e9e325a29ffdc19c65c95fe0a1df 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:16:02,041 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-09T21:16:02,042 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-09T21:16:02,042 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:02,042 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:02,042 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,042 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,042 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-09T21:16:02,042 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,042 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,042 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:02,042 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:02,042 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,043 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:02,043 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=A 2024-11-09T21:16:02,045 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-09T21:16:02,045 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-09T21:16:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:02,045 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=B 2024-11-09T21:16:02,045 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:02,045 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:02,045 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK ac01e9e325a29ffdc19c65c95fe0a1df, store=C 2024-11-09T21:16:02,045 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:02,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:02,045 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:02,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/7bd24f5822a9401ebcc94b7cd1a20684 is 50, key is test_row_0/A:col10/1731186962041/Put/seqid=0 2024-11-09T21:16:02,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741939_1115 (size=12301) 2024-11-09T21:16:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-09T21:16:02,268 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 120 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 118 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-09T21:16:02,269 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2350 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2319 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1073 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3218 rows 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1070 2024-11-09T21:16:02,270 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3203 rows 2024-11-09T21:16:02,271 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:16:02,271 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:16:02,271 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:02,272 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:02,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T21:16:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T21:16:02,284 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186962284"}]},"ts":"1731186962284"} 2024-11-09T21:16:02,286 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T21:16:02,286 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T21:16:02,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:16:02,292 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, UNASSIGN}] 2024-11-09T21:16:02,293 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, UNASSIGN 2024-11-09T21:16:02,295 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ac01e9e325a29ffdc19c65c95fe0a1df, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:02,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, UNASSIGN because future has completed 2024-11-09T21:16:02,297 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:16:02,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:16:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T21:16:02,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/7bd24f5822a9401ebcc94b7cd1a20684 2024-11-09T21:16:02,457 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:16:02,457 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:16:02,458 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing ac01e9e325a29ffdc19c65c95fe0a1df, disabling compactions & flushes 2024-11-09T21:16:02,459 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:02,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/61596ef8aec7455ca3a4b0d2c87ec9e2 is 50, key is test_row_0/B:col10/1731186962041/Put/seqid=0 2024-11-09T21:16:02,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741940_1116 (size=12301) 2024-11-09T21:16:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T21:16:02,873 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/61596ef8aec7455ca3a4b0d2c87ec9e2 2024-11-09T21:16:02,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/96b53ce471c04601b35cbfa244fcb9c9 is 50, key is test_row_0/C:col10/1731186962041/Put/seqid=0 2024-11-09T21:16:02,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741941_1117 (size=12301) 2024-11-09T21:16:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T21:16:03,222 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:16:03,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=618 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/96b53ce471c04601b35cbfa244fcb9c9 2024-11-09T21:16:03,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/A/7bd24f5822a9401ebcc94b7cd1a20684 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7bd24f5822a9401ebcc94b7cd1a20684 2024-11-09T21:16:03,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7bd24f5822a9401ebcc94b7cd1a20684, entries=150, sequenceid=618, filesize=12.0 K 2024-11-09T21:16:03,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/B/61596ef8aec7455ca3a4b0d2c87ec9e2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/61596ef8aec7455ca3a4b0d2c87ec9e2 2024-11-09T21:16:03,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/61596ef8aec7455ca3a4b0d2c87ec9e2, entries=150, sequenceid=618, filesize=12.0 K 2024-11-09T21:16:03,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/.tmp/C/96b53ce471c04601b35cbfa244fcb9c9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/96b53ce471c04601b35cbfa244fcb9c9 2024-11-09T21:16:03,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/96b53ce471c04601b35cbfa244fcb9c9, entries=150, sequenceid=618, filesize=12.0 K 2024-11-09T21:16:03,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for ac01e9e325a29ffdc19c65c95fe0a1df in 1278ms, sequenceid=618, compaction requested=true 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for ac01e9e325a29ffdc19c65c95fe0a1df: 2024-11-09T21:16:03,319 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:03,319 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:03,319 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. after waiting 0 ms 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:03,319 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store ac01e9e325a29ffdc19c65c95fe0a1df:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:03,319 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. because compaction request was cancelled 2024-11-09T21:16:03,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T21:16:03,319 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:A 2024-11-09T21:16:03,319 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. because compaction request was cancelled 2024-11-09T21:16:03,319 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:B 2024-11-09T21:16:03,319 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. because compaction request was cancelled 2024-11-09T21:16:03,319 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: ac01e9e325a29ffdc19c65c95fe0a1df:C 2024-11-09T21:16:03,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d0733887c9d94b509b196e4e0d485015, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/57a3de26846c444aa01e245f69fc1cbe, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/652861cfdb874117bb465ae5eee50a9e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/8058b1be20cf482b8650bc252b8a353a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/a1245dde58084cca87b3f98006f287d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d4c23e142d5d4befa1f69ec95a0ca297, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/08e517ea8d7d41799c47d42aaa404d53, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/65226c0956114d529d91805a62ebca20, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/ffa65120ddfd45d3b6eb7ea341807813, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/01c144b4000f4cb0a7439ea8b2a9841b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7492cc05e72b4839978740478226e014, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d5f2472d2354a6a83c2787ee91ddb3e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b230acaf2c75400bae89adcd54c1e555, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d20ac74e94b948acaf2fa8f4a60d4ac6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d6987dc8d6f43828825af97988d27ea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/fafcb343dd3a42d9a14823cf2b1aca7c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/14114e03c1ed45e8a5ad3f06546a833a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/f2ff97e31da64fd684dc57edb9ee6230, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1ca472dc3c6244e2bf4edc297179a990, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/147e532c66db4264ab30f8a780757360, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c295c8ae498c46e8a8923c65db6c51e6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b731b5e6e24c49d986e5ed0164bfcdfd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/efbabfb1da59490397503b6acb572cea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/53255a70bc094466b7edb55b38d05c6f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b7106b10aa234bc5806b17cbb5d8a200, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b1241914fbf34027a28b200172facdf6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e81fbc571285455394687d5922f4ff64, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/409248b18c484833aebf3498fdac6cf7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d8e67afd3bb7416ead52f3901a1746a4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e2892405c3cf4b07a1b1d34607170be8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/624bb8176da945e08234f933ecf0c59d] to archive 2024-11-09T21:16:03,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:16:03,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d0733887c9d94b509b196e4e0d485015 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d0733887c9d94b509b196e4e0d485015 2024-11-09T21:16:03,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/57a3de26846c444aa01e245f69fc1cbe to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/57a3de26846c444aa01e245f69fc1cbe 2024-11-09T21:16:03,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/652861cfdb874117bb465ae5eee50a9e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/652861cfdb874117bb465ae5eee50a9e 2024-11-09T21:16:03,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/8058b1be20cf482b8650bc252b8a353a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/8058b1be20cf482b8650bc252b8a353a 2024-11-09T21:16:03,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/a1245dde58084cca87b3f98006f287d1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/a1245dde58084cca87b3f98006f287d1 2024-11-09T21:16:03,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d4c23e142d5d4befa1f69ec95a0ca297 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d4c23e142d5d4befa1f69ec95a0ca297 2024-11-09T21:16:03,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/08e517ea8d7d41799c47d42aaa404d53 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/08e517ea8d7d41799c47d42aaa404d53 2024-11-09T21:16:03,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/65226c0956114d529d91805a62ebca20 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/65226c0956114d529d91805a62ebca20 2024-11-09T21:16:03,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/ffa65120ddfd45d3b6eb7ea341807813 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/ffa65120ddfd45d3b6eb7ea341807813 2024-11-09T21:16:03,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/01c144b4000f4cb0a7439ea8b2a9841b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/01c144b4000f4cb0a7439ea8b2a9841b 2024-11-09T21:16:03,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7492cc05e72b4839978740478226e014 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7492cc05e72b4839978740478226e014 2024-11-09T21:16:03,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d5f2472d2354a6a83c2787ee91ddb3e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d5f2472d2354a6a83c2787ee91ddb3e 2024-11-09T21:16:03,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b230acaf2c75400bae89adcd54c1e555 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b230acaf2c75400bae89adcd54c1e555 2024-11-09T21:16:03,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d20ac74e94b948acaf2fa8f4a60d4ac6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d20ac74e94b948acaf2fa8f4a60d4ac6 2024-11-09T21:16:03,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d6987dc8d6f43828825af97988d27ea to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1d6987dc8d6f43828825af97988d27ea 2024-11-09T21:16:03,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/fafcb343dd3a42d9a14823cf2b1aca7c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/fafcb343dd3a42d9a14823cf2b1aca7c 2024-11-09T21:16:03,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/14114e03c1ed45e8a5ad3f06546a833a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/14114e03c1ed45e8a5ad3f06546a833a 2024-11-09T21:16:03,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/f2ff97e31da64fd684dc57edb9ee6230 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/f2ff97e31da64fd684dc57edb9ee6230 2024-11-09T21:16:03,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1ca472dc3c6244e2bf4edc297179a990 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/1ca472dc3c6244e2bf4edc297179a990 2024-11-09T21:16:03,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/147e532c66db4264ab30f8a780757360 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/147e532c66db4264ab30f8a780757360 2024-11-09T21:16:03,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c295c8ae498c46e8a8923c65db6c51e6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c295c8ae498c46e8a8923c65db6c51e6 2024-11-09T21:16:03,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b731b5e6e24c49d986e5ed0164bfcdfd to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b731b5e6e24c49d986e5ed0164bfcdfd 2024-11-09T21:16:03,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/efbabfb1da59490397503b6acb572cea to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/efbabfb1da59490397503b6acb572cea 2024-11-09T21:16:03,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/53255a70bc094466b7edb55b38d05c6f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/53255a70bc094466b7edb55b38d05c6f 2024-11-09T21:16:03,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b7106b10aa234bc5806b17cbb5d8a200 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b7106b10aa234bc5806b17cbb5d8a200 2024-11-09T21:16:03,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b1241914fbf34027a28b200172facdf6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/b1241914fbf34027a28b200172facdf6 2024-11-09T21:16:03,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e81fbc571285455394687d5922f4ff64 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e81fbc571285455394687d5922f4ff64 2024-11-09T21:16:03,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/409248b18c484833aebf3498fdac6cf7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/409248b18c484833aebf3498fdac6cf7 2024-11-09T21:16:03,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d8e67afd3bb7416ead52f3901a1746a4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/d8e67afd3bb7416ead52f3901a1746a4 2024-11-09T21:16:03,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e2892405c3cf4b07a1b1d34607170be8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/e2892405c3cf4b07a1b1d34607170be8 2024-11-09T21:16:03,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/624bb8176da945e08234f933ecf0c59d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/624bb8176da945e08234f933ecf0c59d 2024-11-09T21:16:03,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ab4eb4bf6e8b4467a459e73497c5f28e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c6a15fc74f50433da7f7db2b5644250b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/99fb77982e2d43758d36022812ef222c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cb0f7681472849a5a7d3c14f34f62641, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c4d97f15708f4a2daa35987fe346ba90, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/75ef6062863c4de7bc7a997262fabbb2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/38838d90cdb8443eae3fac2e9c683dd2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/e6bfac905bd14b818347a9762b4905af, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c33b9db369934af1b65dc7c799b2fe57, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/4f24b58f98e24ae6918c02724fd2839a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/708dd1f8e3524ed9b7323c3162d75e25, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/34810630094841dbb389f9d0f6908755, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cadf4ccdc8b34558b5f5c43977caef86, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/73fa3721bd80485c8fb7e5d9f8936371, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/2d576dd8a0b54a91a95741e86c302ba2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/aacec4063b824f12a90da30fbafd90de, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/76fd4dba43c5480bbacdb0324d24d9c7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f6c36c6938184c489c68d169e9be2d59, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cbf037b8c084473aa33b0ce2e6097190, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/0b10e03257344d5d97964f9acdceab15, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/66b9b971423948fca2a5062e7065b1e3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/9d9b520508bc4b6aa24d35e86454f6cd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/bfe6a44787214f0f9aec9a9a8fc91e54, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/69a9ebc9ac9b4306b1a7fef3d066b91c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/baf11684c6e44afba9aa833f763ebff4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f03c9ee236d0436caea7869340db6d7a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/d3e30f8819484369b1efd8623a0c107d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ea570b1f6dec4210b5d51fd0f4f894a9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/fd7d9b2e49764a4583cb00b24d6fef7f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c5364be58db34fb1bfcfc7826245683f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/3cee9d6e1f6a44e38ccd74b02f6a5e30] to archive 2024-11-09T21:16:03,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:16:03,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ab4eb4bf6e8b4467a459e73497c5f28e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ab4eb4bf6e8b4467a459e73497c5f28e 2024-11-09T21:16:03,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c6a15fc74f50433da7f7db2b5644250b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c6a15fc74f50433da7f7db2b5644250b 2024-11-09T21:16:03,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/99fb77982e2d43758d36022812ef222c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/99fb77982e2d43758d36022812ef222c 2024-11-09T21:16:03,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cb0f7681472849a5a7d3c14f34f62641 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cb0f7681472849a5a7d3c14f34f62641 2024-11-09T21:16:03,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c4d97f15708f4a2daa35987fe346ba90 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c4d97f15708f4a2daa35987fe346ba90 2024-11-09T21:16:03,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/75ef6062863c4de7bc7a997262fabbb2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/75ef6062863c4de7bc7a997262fabbb2 2024-11-09T21:16:03,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/38838d90cdb8443eae3fac2e9c683dd2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/38838d90cdb8443eae3fac2e9c683dd2 2024-11-09T21:16:03,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/e6bfac905bd14b818347a9762b4905af to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/e6bfac905bd14b818347a9762b4905af 2024-11-09T21:16:03,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c33b9db369934af1b65dc7c799b2fe57 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c33b9db369934af1b65dc7c799b2fe57 2024-11-09T21:16:03,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/4f24b58f98e24ae6918c02724fd2839a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/4f24b58f98e24ae6918c02724fd2839a 2024-11-09T21:16:03,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/708dd1f8e3524ed9b7323c3162d75e25 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/708dd1f8e3524ed9b7323c3162d75e25 2024-11-09T21:16:03,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/34810630094841dbb389f9d0f6908755 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/34810630094841dbb389f9d0f6908755 2024-11-09T21:16:03,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cadf4ccdc8b34558b5f5c43977caef86 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cadf4ccdc8b34558b5f5c43977caef86 2024-11-09T21:16:03,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/73fa3721bd80485c8fb7e5d9f8936371 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/73fa3721bd80485c8fb7e5d9f8936371 2024-11-09T21:16:03,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/2d576dd8a0b54a91a95741e86c302ba2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/2d576dd8a0b54a91a95741e86c302ba2 2024-11-09T21:16:03,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/aacec4063b824f12a90da30fbafd90de to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/aacec4063b824f12a90da30fbafd90de 2024-11-09T21:16:03,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/76fd4dba43c5480bbacdb0324d24d9c7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/76fd4dba43c5480bbacdb0324d24d9c7 2024-11-09T21:16:03,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f6c36c6938184c489c68d169e9be2d59 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f6c36c6938184c489c68d169e9be2d59 2024-11-09T21:16:03,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cbf037b8c084473aa33b0ce2e6097190 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/cbf037b8c084473aa33b0ce2e6097190 2024-11-09T21:16:03,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/0b10e03257344d5d97964f9acdceab15 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/0b10e03257344d5d97964f9acdceab15 2024-11-09T21:16:03,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/66b9b971423948fca2a5062e7065b1e3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/66b9b971423948fca2a5062e7065b1e3 2024-11-09T21:16:03,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/9d9b520508bc4b6aa24d35e86454f6cd to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/9d9b520508bc4b6aa24d35e86454f6cd 2024-11-09T21:16:03,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/bfe6a44787214f0f9aec9a9a8fc91e54 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/bfe6a44787214f0f9aec9a9a8fc91e54 2024-11-09T21:16:03,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/69a9ebc9ac9b4306b1a7fef3d066b91c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/69a9ebc9ac9b4306b1a7fef3d066b91c 2024-11-09T21:16:03,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/baf11684c6e44afba9aa833f763ebff4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/baf11684c6e44afba9aa833f763ebff4 2024-11-09T21:16:03,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f03c9ee236d0436caea7869340db6d7a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/f03c9ee236d0436caea7869340db6d7a 2024-11-09T21:16:03,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/d3e30f8819484369b1efd8623a0c107d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/d3e30f8819484369b1efd8623a0c107d 2024-11-09T21:16:03,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ea570b1f6dec4210b5d51fd0f4f894a9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/ea570b1f6dec4210b5d51fd0f4f894a9 2024-11-09T21:16:03,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/fd7d9b2e49764a4583cb00b24d6fef7f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/fd7d9b2e49764a4583cb00b24d6fef7f 2024-11-09T21:16:03,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c5364be58db34fb1bfcfc7826245683f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/c5364be58db34fb1bfcfc7826245683f 2024-11-09T21:16:03,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/3cee9d6e1f6a44e38ccd74b02f6a5e30 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/3cee9d6e1f6a44e38ccd74b02f6a5e30 2024-11-09T21:16:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T21:16:03,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3a18f9be23bf46938682d9e2adada014, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ab77949e1a124651983d3bbaf8911908, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/acbbc82559a940719ca87fdd645edb1f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7288d2a2e5e244da9d807f1d14842776, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c2ae2be3d4ce405cafc304801fc66056, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7c77915d80834933a3c3ac951955c613, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/2747bdb68eee4f8c8db199416373191a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/b49f908889c34492a82bd5c931fa112d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/a1b40204674745c6aa7d31f8ee4f1804, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/65b6b9d8e5bf485da1ee1ccdf80714e5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e3139bd5a22f4797a9eb8fe23c13903a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3ed51fd076c94dd1bf659750633da8fb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/d28f1709234647509fc43ba724dd7bd4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/476a900f31564508920c484fdd26eb25, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c779c31c50ce47ca977cde1cfbd79480, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ed94f44ea04347cab1366b63b4f0742b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/abd16081fea14b0c9d7564d8d3382a9f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f26c528f6f4d434d96906e005398b834, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/76ae0d25114e47b485f80a1c572a2ab1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e53478e3578f49d89f7899eb9974cbab, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f8cc069a6de2447ea8233903e2e8fc64, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bc8edd93fdf3485696909eb7fdfc8c74, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e642d9833c814033a62fb7a776679de4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/cfa9d5e3c8144b7a982b0e423797329f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/eb68a02aef734bdeba97c7839f6c12c6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f85dc07644d047458e3dc1bb8784c609, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1640102f2da9460aba36b2d19e33dd64, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bed554bdaaf7448f802d70ae996068fb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/4a02f0505c4748c0b917e9e27aae04c9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/0d4b91d9f20d48908a4735b438b05be8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f37afba73348463eab9cda5fc01349bc] to archive 2024-11-09T21:16:03,429 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:16:03,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3a18f9be23bf46938682d9e2adada014 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3a18f9be23bf46938682d9e2adada014 2024-11-09T21:16:03,431 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ab77949e1a124651983d3bbaf8911908 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ab77949e1a124651983d3bbaf8911908 2024-11-09T21:16:03,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/acbbc82559a940719ca87fdd645edb1f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/acbbc82559a940719ca87fdd645edb1f 2024-11-09T21:16:03,434 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7288d2a2e5e244da9d807f1d14842776 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7288d2a2e5e244da9d807f1d14842776 2024-11-09T21:16:03,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c2ae2be3d4ce405cafc304801fc66056 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c2ae2be3d4ce405cafc304801fc66056 2024-11-09T21:16:03,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7c77915d80834933a3c3ac951955c613 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/7c77915d80834933a3c3ac951955c613 2024-11-09T21:16:03,437 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/2747bdb68eee4f8c8db199416373191a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/2747bdb68eee4f8c8db199416373191a 2024-11-09T21:16:03,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/b49f908889c34492a82bd5c931fa112d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/b49f908889c34492a82bd5c931fa112d 2024-11-09T21:16:03,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/a1b40204674745c6aa7d31f8ee4f1804 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/a1b40204674745c6aa7d31f8ee4f1804 2024-11-09T21:16:03,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/65b6b9d8e5bf485da1ee1ccdf80714e5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/65b6b9d8e5bf485da1ee1ccdf80714e5 2024-11-09T21:16:03,442 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e3139bd5a22f4797a9eb8fe23c13903a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e3139bd5a22f4797a9eb8fe23c13903a 2024-11-09T21:16:03,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3ed51fd076c94dd1bf659750633da8fb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/3ed51fd076c94dd1bf659750633da8fb 2024-11-09T21:16:03,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/d28f1709234647509fc43ba724dd7bd4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/d28f1709234647509fc43ba724dd7bd4 2024-11-09T21:16:03,445 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/476a900f31564508920c484fdd26eb25 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/476a900f31564508920c484fdd26eb25 2024-11-09T21:16:03,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c779c31c50ce47ca977cde1cfbd79480 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c779c31c50ce47ca977cde1cfbd79480 2024-11-09T21:16:03,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ed94f44ea04347cab1366b63b4f0742b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/ed94f44ea04347cab1366b63b4f0742b 2024-11-09T21:16:03,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/abd16081fea14b0c9d7564d8d3382a9f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/abd16081fea14b0c9d7564d8d3382a9f 2024-11-09T21:16:03,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f26c528f6f4d434d96906e005398b834 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f26c528f6f4d434d96906e005398b834 2024-11-09T21:16:03,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/76ae0d25114e47b485f80a1c572a2ab1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/76ae0d25114e47b485f80a1c572a2ab1 2024-11-09T21:16:03,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e53478e3578f49d89f7899eb9974cbab to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e53478e3578f49d89f7899eb9974cbab 2024-11-09T21:16:03,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f8cc069a6de2447ea8233903e2e8fc64 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f8cc069a6de2447ea8233903e2e8fc64 2024-11-09T21:16:03,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bc8edd93fdf3485696909eb7fdfc8c74 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bc8edd93fdf3485696909eb7fdfc8c74 2024-11-09T21:16:03,454 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e642d9833c814033a62fb7a776679de4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/e642d9833c814033a62fb7a776679de4 2024-11-09T21:16:03,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/cfa9d5e3c8144b7a982b0e423797329f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/cfa9d5e3c8144b7a982b0e423797329f 2024-11-09T21:16:03,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/eb68a02aef734bdeba97c7839f6c12c6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/eb68a02aef734bdeba97c7839f6c12c6 2024-11-09T21:16:03,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f85dc07644d047458e3dc1bb8784c609 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f85dc07644d047458e3dc1bb8784c609 2024-11-09T21:16:03,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1640102f2da9460aba36b2d19e33dd64 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1640102f2da9460aba36b2d19e33dd64 2024-11-09T21:16:03,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bed554bdaaf7448f802d70ae996068fb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/bed554bdaaf7448f802d70ae996068fb 2024-11-09T21:16:03,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/4a02f0505c4748c0b917e9e27aae04c9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/4a02f0505c4748c0b917e9e27aae04c9 2024-11-09T21:16:03,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/0d4b91d9f20d48908a4735b438b05be8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/0d4b91d9f20d48908a4735b438b05be8 2024-11-09T21:16:03,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f37afba73348463eab9cda5fc01349bc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/f37afba73348463eab9cda5fc01349bc 2024-11-09T21:16:03,467 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/recovered.edits/621.seqid, newMaxSeqId=621, maxSeqId=1 2024-11-09T21:16:03,468 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df. 2024-11-09T21:16:03,468 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for ac01e9e325a29ffdc19c65c95fe0a1df: Waiting for close lock at 1731186962458Running coprocessor pre-close hooks at 1731186962458Disabling compacts and flushes for region at 1731186962458Disabling writes for close at 1731186963319 (+861 ms)Writing region close event to WAL at 1731186963464 (+145 ms)Running coprocessor post-close hooks at 1731186963468 (+4 ms)Closed at 1731186963468 2024-11-09T21:16:03,470 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:16:03,471 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=ac01e9e325a29ffdc19c65c95fe0a1df, regionState=CLOSED 2024-11-09T21:16:03,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:16:03,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-09T21:16:03,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure ac01e9e325a29ffdc19c65c95fe0a1df, server=9514d6d5e5be,38413,1731186934857 in 1.1760 sec 2024-11-09T21:16:03,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-09T21:16:03,477 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=ac01e9e325a29ffdc19c65c95fe0a1df, UNASSIGN in 1.1830 sec 2024-11-09T21:16:03,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-09T21:16:03,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1900 sec 2024-11-09T21:16:03,482 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186963482"}]},"ts":"1731186963482"} 2024-11-09T21:16:03,484 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T21:16:03,484 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T21:16:03,486 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.2050 sec 2024-11-09T21:16:04,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-09T21:16:04,437 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:04,440 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T21:16:04,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,449 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T21:16:04,450 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T21:16:04,455 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:16:04,459 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/recovered.edits] 2024-11-09T21:16:04,462 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7bd24f5822a9401ebcc94b7cd1a20684 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/7bd24f5822a9401ebcc94b7cd1a20684 2024-11-09T21:16:04,463 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/abf29577c76f4179a98f8c7095294c9c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/abf29577c76f4179a98f8c7095294c9c 2024-11-09T21:16:04,464 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c04ca1bdc82640b1a108d0c3397ef1ba to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c04ca1bdc82640b1a108d0c3397ef1ba 2024-11-09T21:16:04,465 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c4a1c18f6e8d49a5be2182fb817809b5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/A/c4a1c18f6e8d49a5be2182fb817809b5 2024-11-09T21:16:04,468 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/1e94fe0a7eaf40edbe3907de55020d01 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/1e94fe0a7eaf40edbe3907de55020d01 2024-11-09T21:16:04,469 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/26a1a280d1474ff68bca4bf0d0625363 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/26a1a280d1474ff68bca4bf0d0625363 2024-11-09T21:16:04,470 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/61596ef8aec7455ca3a4b0d2c87ec9e2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/61596ef8aec7455ca3a4b0d2c87ec9e2 2024-11-09T21:16:04,471 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/8adabd51abe54ede9c33184bf01eed45 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/B/8adabd51abe54ede9c33184bf01eed45 2024-11-09T21:16:04,474 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1e19f5f9daa341f3998a32c0aaca3fed to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/1e19f5f9daa341f3998a32c0aaca3fed 2024-11-09T21:16:04,475 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/68bfb89cec20429398b3130d817f0fca to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/68bfb89cec20429398b3130d817f0fca 2024-11-09T21:16:04,476 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/96b53ce471c04601b35cbfa244fcb9c9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/96b53ce471c04601b35cbfa244fcb9c9 2024-11-09T21:16:04,477 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c27916042fed423fa001702605f2bca5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/C/c27916042fed423fa001702605f2bca5 2024-11-09T21:16:04,480 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/recovered.edits/621.seqid to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df/recovered.edits/621.seqid 2024-11-09T21:16:04,480 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/ac01e9e325a29ffdc19c65c95fe0a1df 2024-11-09T21:16:04,480 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T21:16:04,486 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,490 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-09T21:16:04,497 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T21:16:04,533 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T21:16:04,536 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,536 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T21:16:04,537 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731186964536"}]},"ts":"9223372036854775807"} 2024-11-09T21:16:04,542 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T21:16:04,542 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => ac01e9e325a29ffdc19c65c95fe0a1df, NAME => 'TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T21:16:04,542 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T21:16:04,543 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731186964543"}]},"ts":"9223372036854775807"} 2024-11-09T21:16:04,549 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T21:16:04,551 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 110 msec 2024-11-09T21:16:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-09T21:16:04,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T21:16:04,557 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:04,573 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1645226176_22 at /127.0.0.1:46340 [Waiting for operation #106] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1645226176_22 at /127.0.0.1:59368 [Waiting for operation #282] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9514d6d5e5be:38413-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-71581069_22 at /127.0.0.1:58510 [Waiting for operation #263] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=363 (was 204) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3791 (was 3627) - AvailableMemoryMB LEAK? - 2024-11-09T21:16:04,584 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=363, ProcessCount=11, AvailableMemoryMB=3791 2024-11-09T21:16:04,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:16:04,586 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:16:04,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:04,588 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:16:04,588 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:04,588 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-09T21:16:04,589 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:16:04,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-09T21:16:04,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741942_1118 (size=1061) 2024-11-09T21:16:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-09T21:16:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-09T21:16:04,997 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:16:05,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741943_1119 (size=53) 2024-11-09T21:16:05,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-09T21:16:05,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:16:05,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing aeea20555c2d70c5adac9bfee3363c0e, disabling compactions & flushes 2024-11-09T21:16:05,406 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. after waiting 0 ms 2024-11-09T21:16:05,406 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,406 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,407 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for aeea20555c2d70c5adac9bfee3363c0e: Waiting for close lock at 1731186965406Disabling compacts and flushes for region at 1731186965406Disabling writes for close at 1731186965406Writing region close event to WAL at 1731186965406Closed at 1731186965406 2024-11-09T21:16:05,409 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:16:05,410 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731186965409"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186965409"}]},"ts":"1731186965409"} 2024-11-09T21:16:05,413 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:16:05,415 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:16:05,415 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186965415"}]},"ts":"1731186965415"} 2024-11-09T21:16:05,417 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T21:16:05,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, ASSIGN}] 2024-11-09T21:16:05,418 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, ASSIGN 2024-11-09T21:16:05,419 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, ASSIGN; state=OFFLINE, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=false 2024-11-09T21:16:05,570 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:05,572 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, ASSIGN because future has completed 2024-11-09T21:16:05,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:16:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-09T21:16:05,728 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,728 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:16:05,728 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,728 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:16:05,728 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,728 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,730 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,731 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:05,732 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeea20555c2d70c5adac9bfee3363c0e columnFamilyName A 2024-11-09T21:16:05,733 DEBUG [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:05,733 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(327): Store=aeea20555c2d70c5adac9bfee3363c0e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:05,733 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,735 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:05,735 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeea20555c2d70c5adac9bfee3363c0e columnFamilyName B 2024-11-09T21:16:05,735 DEBUG [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:05,735 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(327): Store=aeea20555c2d70c5adac9bfee3363c0e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:05,736 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,737 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:05,737 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeea20555c2d70c5adac9bfee3363c0e columnFamilyName C 2024-11-09T21:16:05,737 DEBUG [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:05,738 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(327): Store=aeea20555c2d70c5adac9bfee3363c0e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:05,738 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,738 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,739 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,740 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,741 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,741 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:16:05,743 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,745 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:16:05,746 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened aeea20555c2d70c5adac9bfee3363c0e; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69302526, jitterRate=0.03268811106681824}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:16:05,746 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:05,748 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for aeea20555c2d70c5adac9bfee3363c0e: Running coprocessor pre-open hook at 1731186965729Writing region info on filesystem at 1731186965729Initializing all the Stores at 1731186965729Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186965729Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186965730 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186965730Cleaning up temporary data from old regions at 1731186965741 (+11 ms)Running coprocessor post-open hooks at 1731186965746 (+5 ms)Region opened successfully at 1731186965748 (+2 ms) 2024-11-09T21:16:05,749 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., pid=34, masterSystemTime=1731186965724 2024-11-09T21:16:05,751 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,751 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:05,752 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:05,753 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:16:05,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-09T21:16:05,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 in 182 msec 2024-11-09T21:16:05,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-09T21:16:05,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, ASSIGN in 339 msec 2024-11-09T21:16:05,758 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:16:05,759 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186965758"}]},"ts":"1731186965758"} 2024-11-09T21:16:05,760 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T21:16:05,761 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:16:05,762 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1750 sec 2024-11-09T21:16:06,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-09T21:16:06,738 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:06,740 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42e30321, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:06,740 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:06,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:06,744 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e84579, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:06,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:06,746 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:06,748 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41882, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:06,749 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a869d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:06,749 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:06,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:06,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35474, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:06,753 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7a9498cf 2024-11-09T21:16:06,754 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:16:06,755 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:16:06,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:16:06,760 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:16:06,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:06,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741944_1120 (size=1097) 2024-11-09T21:16:07,181 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T21:16:07,181 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T21:16:07,185 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:16:07,193 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, REOPEN/MOVE}] 2024-11-09T21:16:07,194 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, REOPEN/MOVE 2024-11-09T21:16:07,196 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,198 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, REOPEN/MOVE because future has completed 2024-11-09T21:16:07,199 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:16:07,199 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:16:07,353 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,353 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:16:07,353 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing aeea20555c2d70c5adac9bfee3363c0e, disabling compactions & flushes 2024-11-09T21:16:07,354 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,354 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,354 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. after waiting 0 ms 2024-11-09T21:16:07,354 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,361 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:16:07,362 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,362 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for aeea20555c2d70c5adac9bfee3363c0e: Waiting for close lock at 1731186967353Running coprocessor pre-close hooks at 1731186967353Disabling compacts and flushes for region at 1731186967353Disabling writes for close at 1731186967354 (+1 ms)Writing region close event to WAL at 1731186967355 (+1 ms)Running coprocessor post-close hooks at 1731186967362 (+7 ms)Closed at 1731186967362 2024-11-09T21:16:07,362 WARN [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: aeea20555c2d70c5adac9bfee3363c0e to self. 2024-11-09T21:16:07,365 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=CLOSED 2024-11-09T21:16:07,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:16:07,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-09T21:16:07,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 in 171 msec 2024-11-09T21:16:07,373 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, REOPEN/MOVE; state=CLOSED, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=true 2024-11-09T21:16:07,523 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,525 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, REOPEN/MOVE because future has completed 2024-11-09T21:16:07,526 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:16:07,688 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,688 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:16:07,688 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,688 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:16:07,688 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,689 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,693 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,694 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:07,699 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeea20555c2d70c5adac9bfee3363c0e columnFamilyName A 2024-11-09T21:16:07,701 DEBUG [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:07,702 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(327): Store=aeea20555c2d70c5adac9bfee3363c0e/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:07,703 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,704 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:07,705 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeea20555c2d70c5adac9bfee3363c0e columnFamilyName B 2024-11-09T21:16:07,705 DEBUG [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:07,705 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(327): Store=aeea20555c2d70c5adac9bfee3363c0e/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:07,705 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,706 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:07,706 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region aeea20555c2d70c5adac9bfee3363c0e columnFamilyName C 2024-11-09T21:16:07,707 DEBUG [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:07,707 INFO [StoreOpener-aeea20555c2d70c5adac9bfee3363c0e-1 {}] regionserver.HStore(327): Store=aeea20555c2d70c5adac9bfee3363c0e/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:07,707 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,707 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,708 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,709 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,710 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,710 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,711 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:16:07,712 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,713 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened aeea20555c2d70c5adac9bfee3363c0e; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60521673, jitterRate=-0.09815679490566254}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:16:07,713 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,714 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for aeea20555c2d70c5adac9bfee3363c0e: Running coprocessor pre-open hook at 1731186967689Writing region info on filesystem at 1731186967689Initializing all the Stores at 1731186967690 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186967690Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186967692 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186967692Cleaning up temporary data from old regions at 1731186967710 (+18 ms)Running coprocessor post-open hooks at 1731186967713 (+3 ms)Region opened successfully at 1731186967714 (+1 ms) 2024-11-09T21:16:07,715 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., pid=39, masterSystemTime=1731186967678 2024-11-09T21:16:07,718 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,718 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,719 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=OPEN, openSeqNum=5, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,720 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:16:07,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-09T21:16:07,723 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 in 196 msec 2024-11-09T21:16:07,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-09T21:16:07,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, REOPEN/MOVE in 530 msec 2024-11-09T21:16:07,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-09T21:16:07,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 542 msec 2024-11-09T21:16:07,731 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 966 msec 2024-11-09T21:16:07,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-09T21:16:07,733 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:07,733 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f9c3dcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,734 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,734 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,735 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eaf2cd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,736 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1953facc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,737 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dde8260, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,737 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,738 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,738 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ecb8e63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,740 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e668ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,741 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab5c81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,741 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,741 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,742 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1968ec37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,743 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,744 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41934, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@754da9f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,745 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea1a430, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,745 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,747 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164066, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,748 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,749 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41960, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68eb0c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fbff363, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,751 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,751 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7af3d1b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,752 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,753 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d3b12af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,755 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@292bddff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,755 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,755 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,756 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6647e715, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,757 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42004, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,758 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74492d72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21a14e5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,760 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@575721, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,760 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,761 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42016, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f667eff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,763 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f17c234, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,763 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,763 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,764 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78fe878c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,764 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,766 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,766 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34e5a04f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@197f7df3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:07,768 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:07,769 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18a3a102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,769 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:07,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:07,770 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:07,771 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:07,771 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dd14a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:07,772 DEBUG [Thread-628 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,772 DEBUG [Thread-626 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,772 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-09T21:16:07,775 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:07,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,775 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T21:16:07,776 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:07,776 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,780 DEBUG [Thread-611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,781 DEBUG [Thread-617 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,781 DEBUG [Thread-624 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,781 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,782 DEBUG [Thread-613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,782 DEBUG [Thread-622 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,782 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,783 DEBUG [Thread-615 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,784 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35512, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35514, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35502, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,787 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35520, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,787 DEBUG [Thread-619 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:07,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,791 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,792 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:07,793 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:07,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,796 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:07,801 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:16:07,804 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:16:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:07,805 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731186967845, max timestamp=1731186967868Num uniques 859; 2024-11-09T21:16:07,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T21:16:07,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731186967845, max timestamp=1731186967868Num uniques 859; 2024-11-09T21:16:07,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T21:16:07,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e89ffd914fe54fad84dda78e31a2535a_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186967795/Put/seqid=0 2024-11-09T21:16:07,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731186967845, max timestamp=1731186967868Num uniques 859; 2024-11-09T21:16:07,878 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T21:16:07,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T21:16:07,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741945_1121 (size=9714) 2024-11-09T21:16:07,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:07,913 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e89ffd914fe54fad84dda78e31a2535a_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e89ffd914fe54fad84dda78e31a2535a_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:07,916 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/89d261a47b7d4f34bf5fd4bfea214804, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:07,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/89d261a47b7d4f34bf5fd4bfea214804 is 175, key is test_row_0/A:col10/1731186967795/Put/seqid=0 2024-11-09T21:16:07,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187027911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187027913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,928 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:07,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187027919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T21:16:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:07,929 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:07,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:07,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:07,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:07,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187027927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:07,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:07,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187027927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:07,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741946_1122 (size=22361) 2024-11-09T21:16:07,941 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:07,952 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/89d261a47b7d4f34bf5fd4bfea214804 2024-11-09T21:16:07,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/079ab2e54d764781bd19387d68b02d57 is 50, key is test_row_0/B:col10/1731186967795/Put/seqid=0 2024-11-09T21:16:08,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741947_1123 (size=9657) 2024-11-09T21:16:08,037 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187028036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187028037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,038 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187028037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,046 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,046 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187028046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187028047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,082 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T21:16:08,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:08,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T21:16:08,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T21:16:08,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:08,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187028246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187028246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,255 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,264 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187028257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187028257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187028257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T21:16:08,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:08,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T21:16:08,407 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/079ab2e54d764781bd19387d68b02d57 2024-11-09T21:16:08,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5e085565f6ae4ab5ace384f9bf884a67 is 50, key is test_row_0/C:col10/1731186967795/Put/seqid=0 2024-11-09T21:16:08,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741948_1124 (size=9657) 2024-11-09T21:16:08,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5e085565f6ae4ab5ace384f9bf884a67 2024-11-09T21:16:08,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/89d261a47b7d4f34bf5fd4bfea214804 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804 2024-11-09T21:16:08,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804, entries=100, sequenceid=14, filesize=21.8 K 2024-11-09T21:16:08,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/079ab2e54d764781bd19387d68b02d57 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/079ab2e54d764781bd19387d68b02d57 2024-11-09T21:16:08,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/079ab2e54d764781bd19387d68b02d57, entries=100, sequenceid=14, filesize=9.4 K 2024-11-09T21:16:08,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5e085565f6ae4ab5ace384f9bf884a67 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5e085565f6ae4ab5ace384f9bf884a67 2024-11-09T21:16:08,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5e085565f6ae4ab5ace384f9bf884a67, entries=100, sequenceid=14, filesize=9.4 K 2024-11-09T21:16:08,518 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for aeea20555c2d70c5adac9bfee3363c0e in 714ms, sequenceid=14, compaction requested=false 2024-11-09T21:16:08,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:08,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-09T21:16:08,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:08,551 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=147.60 KB heapSize=410.77 KB 2024-11-09T21:16:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:08,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:08,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:08,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:08,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187028577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187028577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187028579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187028578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109596902c965454ec2b2f3018382a7eca4_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186967912/Put/seqid=0 2024-11-09T21:16:08,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187028589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,591 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741949_1125 (size=12154) 2024-11-09T21:16:08,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:08,634 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109596902c965454ec2b2f3018382a7eca4_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109596902c965454ec2b2f3018382a7eca4_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/a6f2bb25d2244fa5914b7344caed3320, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:08,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/a6f2bb25d2244fa5914b7344caed3320 is 175, key is test_row_0/A:col10/1731186967912/Put/seqid=0 2024-11-09T21:16:08,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741950_1126 (size=30955) 2024-11-09T21:16:08,699 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/a6f2bb25d2244fa5914b7344caed3320 2024-11-09T21:16:08,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187028696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187028698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/83ef211f26874855bd462a764c6928fd is 50, key is test_row_0/B:col10/1731186967912/Put/seqid=0 2024-11-09T21:16:08,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741951_1127 (size=12001) 2024-11-09T21:16:08,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187028907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187028907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:08,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:08,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T21:16:09,032 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:16:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187029097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187029097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,101 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187029099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,138 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/83ef211f26874855bd462a764c6928fd 2024-11-09T21:16:09,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/09dbe0b53dda40b0b9c10dba98a6088d is 50, key is test_row_0/C:col10/1731186967912/Put/seqid=0 2024-11-09T21:16:09,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741952_1128 (size=12001) 2024-11-09T21:16:09,203 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/09dbe0b53dda40b0b9c10dba98a6088d 2024-11-09T21:16:09,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/a6f2bb25d2244fa5914b7344caed3320 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320 2024-11-09T21:16:09,220 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320, entries=150, sequenceid=50, filesize=30.2 K 2024-11-09T21:16:09,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187029217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,221 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,222 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187029217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/83ef211f26874855bd462a764c6928fd as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/83ef211f26874855bd462a764c6928fd 2024-11-09T21:16:09,238 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/83ef211f26874855bd462a764c6928fd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T21:16:09,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/09dbe0b53dda40b0b9c10dba98a6088d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09dbe0b53dda40b0b9c10dba98a6088d 2024-11-09T21:16:09,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09dbe0b53dda40b0b9c10dba98a6088d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T21:16:09,257 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~410.72 KB/420576, currentSize=40.25 KB/41220 for aeea20555c2d70c5adac9bfee3363c0e in 706ms, sequenceid=50, compaction requested=false 2024-11-09T21:16:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:09,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-09T21:16:09,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-09T21:16:09,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-09T21:16:09,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4830 sec 2024-11-09T21:16:09,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.4910 sec 2024-11-09T21:16:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:09,741 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:16:09,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:09,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:09,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:09,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:09,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:09,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:09,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ab9b1c46ec874f75b124776c3f21addd_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:09,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741953_1129 (size=12154) 2024-11-09T21:16:09,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187029823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731186969800, max timestamp=1731186969823Num uniques 859; 2024-11-09T21:16:09,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731186969800, max timestamp=1731186969823Num uniques 859; 2024-11-09T21:16:09,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T21:16:09,827 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-09T21:16:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731186969807, max timestamp=1731186969823Num uniques 859; 2024-11-09T21:16:09,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T21:16:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-09T21:16:09,927 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:09,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:09,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-09T21:16:09,932 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T21:16:09,935 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:09,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:09,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187029966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187029968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T21:16:10,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187030076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187030076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,084 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T21:16:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,125 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187030116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187030116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,126 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187030117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,126 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,127 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,127 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,128 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,215 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:10,222 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ab9b1c46ec874f75b124776c3f21addd_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ab9b1c46ec874f75b124776c3f21addd_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:10,223 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/79775f79641944fe9de2348814702201, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:10,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/79775f79641944fe9de2348814702201 is 175, key is test_row_0/A:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:10,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T21:16:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:10,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,245 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T21:16:10,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741954_1130 (size=30955) 2024-11-09T21:16:10,270 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/79775f79641944fe9de2348814702201 2024-11-09T21:16:10,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/5fd33241b74941edb85ade8ebc0c9cab is 50, key is test_row_0/B:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:10,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187030287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187030296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,306 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741955_1131 (size=12001) 2024-11-09T21:16:10,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/5fd33241b74941edb85ade8ebc0c9cab 2024-11-09T21:16:10,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/6d12356938864b07859f72916d6bbe63 is 50, key is test_row_0/C:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:10,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741956_1132 (size=12001) 2024-11-09T21:16:10,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T21:16:10,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:10,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T21:16:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T21:16:10,615 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187030607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:10,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187030617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:10,626 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:10,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T21:16:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:10,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:10,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/6d12356938864b07859f72916d6bbe63 2024-11-09T21:16:10,762 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/79775f79641944fe9de2348814702201 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201 2024-11-09T21:16:10,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201, entries=150, sequenceid=61, filesize=30.2 K 2024-11-09T21:16:10,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/5fd33241b74941edb85ade8ebc0c9cab as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5fd33241b74941edb85ade8ebc0c9cab 2024-11-09T21:16:10,783 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5fd33241b74941edb85ade8ebc0c9cab, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:16:10,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/6d12356938864b07859f72916d6bbe63 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/6d12356938864b07859f72916d6bbe63 2024-11-09T21:16:10,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/6d12356938864b07859f72916d6bbe63, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:16:10,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for aeea20555c2d70c5adac9bfee3363c0e in 1054ms, sequenceid=61, compaction requested=true 2024-11-09T21:16:10,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:10,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:10,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:10,796 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:10,796 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:10,797 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 84271 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:10,797 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:10,797 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,797 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=82.3 K 2024-11-09T21:16:10,797 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,798 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201] 2024-11-09T21:16:10,800 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:10,800 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:10,800 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,801 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/079ab2e54d764781bd19387d68b02d57, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/83ef211f26874855bd462a764c6928fd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5fd33241b74941edb85ade8ebc0c9cab] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=32.9 K 2024-11-09T21:16:10,801 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d261a47b7d4f34bf5fd4bfea214804, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731186967795 2024-11-09T21:16:10,802 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 079ab2e54d764781bd19387d68b02d57, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731186967795 2024-11-09T21:16:10,802 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6f2bb25d2244fa5914b7344caed3320, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731186967912 2024-11-09T21:16:10,802 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79775f79641944fe9de2348814702201, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186968566 2024-11-09T21:16:10,802 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 83ef211f26874855bd462a764c6928fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731186967912 2024-11-09T21:16:10,803 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fd33241b74941edb85ade8ebc0c9cab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186968566 2024-11-09T21:16:10,836 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:10,840 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#115 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:10,841 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/1afa62833cfe4817a300748ce89bcc39 is 50, key is test_row_0/B:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:10,845 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411090308d3fd1fba4a03b29f48d62dc96017_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:10,857 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411090308d3fd1fba4a03b29f48d62dc96017_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:10,858 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090308d3fd1fba4a03b29f48d62dc96017_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:10,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-09T21:16:10,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,865 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=127.08 KB heapSize=354.77 KB 2024-11-09T21:16:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:10,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:10,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741957_1133 (size=12104) 2024-11-09T21:16:10,896 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/1afa62833cfe4817a300748ce89bcc39 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/1afa62833cfe4817a300748ce89bcc39 2024-11-09T21:16:10,905 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into 1afa62833cfe4817a300748ce89bcc39(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:10,905 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:10,905 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186970796; duration=0sec 2024-11-09T21:16:10,906 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:10,906 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:10,906 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:10,907 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:10,908 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:10,908 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:10,909 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5e085565f6ae4ab5ace384f9bf884a67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09dbe0b53dda40b0b9c10dba98a6088d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/6d12356938864b07859f72916d6bbe63] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=32.9 K 2024-11-09T21:16:10,910 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e085565f6ae4ab5ace384f9bf884a67, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731186967795 2024-11-09T21:16:10,911 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 09dbe0b53dda40b0b9c10dba98a6088d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731186967912 2024-11-09T21:16:10,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109becfc144d0ee4c22b1b73230f3de4cce_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186969900/Put/seqid=0 2024-11-09T21:16:10,913 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d12356938864b07859f72916d6bbe63, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186968566 2024-11-09T21:16:10,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741958_1134 (size=4469) 2024-11-09T21:16:10,941 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#114 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:10,942 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/fceba0c46f8a4218986a10acf5556aa7 is 175, key is test_row_0/A:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:10,960 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#117 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:10,961 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/8118fb2d8345474784133c664284c45f is 50, key is test_row_0/C:col10/1731186968566/Put/seqid=0 2024-11-09T21:16:10,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741959_1135 (size=12154) 2024-11-09T21:16:10,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:10,999 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109becfc144d0ee4c22b1b73230f3de4cce_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109becfc144d0ee4c22b1b73230f3de4cce_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:11,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/bfac464002ee431a8e7f4d73db6b6fbf, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:11,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/bfac464002ee431a8e7f4d73db6b6fbf is 175, key is test_row_0/A:col10/1731186969900/Put/seqid=0 2024-11-09T21:16:11,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741960_1136 (size=31058) 2024-11-09T21:16:11,030 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/fceba0c46f8a4218986a10acf5556aa7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/fceba0c46f8a4218986a10acf5556aa7 2024-11-09T21:16:11,038 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into fceba0c46f8a4218986a10acf5556aa7(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:11,038 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:11,038 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186970795; duration=0sec 2024-11-09T21:16:11,039 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:11,039 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:11,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741961_1137 (size=12104) 2024-11-09T21:16:11,053 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/8118fb2d8345474784133c664284c45f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8118fb2d8345474784133c664284c45f 2024-11-09T21:16:11,063 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 8118fb2d8345474784133c664284c45f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:11,063 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:11,063 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186970796; duration=0sec 2024-11-09T21:16:11,063 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:11,063 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:11,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741962_1138 (size=30955) 2024-11-09T21:16:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T21:16:11,069 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/bfac464002ee431a8e7f4d73db6b6fbf 2024-11-09T21:16:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/235409bd313b44c9a764308e19e00665 is 50, key is test_row_0/B:col10/1731186969900/Put/seqid=0 2024-11-09T21:16:11,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:11,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:11,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741963_1139 (size=12001) 2024-11-09T21:16:11,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/235409bd313b44c9a764308e19e00665 2024-11-09T21:16:11,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/bae48035fef94befac04361914e6b631 is 50, key is test_row_0/C:col10/1731186969900/Put/seqid=0 2024-11-09T21:16:11,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741964_1140 (size=12001) 2024-11-09T21:16:11,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187031185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,197 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=37.62 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/bae48035fef94befac04361914e6b631 2024-11-09T21:16:11,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187031191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/bfac464002ee431a8e7f4d73db6b6fbf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf 2024-11-09T21:16:11,223 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf, entries=150, sequenceid=95, filesize=30.2 K 2024-11-09T21:16:11,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/235409bd313b44c9a764308e19e00665 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/235409bd313b44c9a764308e19e00665 2024-11-09T21:16:11,231 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/235409bd313b44c9a764308e19e00665, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T21:16:11,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/bae48035fef94befac04361914e6b631 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/bae48035fef94befac04361914e6b631 2024-11-09T21:16:11,246 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/bae48035fef94befac04361914e6b631, entries=150, sequenceid=95, filesize=11.7 K 2024-11-09T21:16:11,247 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~354.73 KB/363240, currentSize=53.67 KB/54960 for aeea20555c2d70c5adac9bfee3363c0e in 382ms, sequenceid=95, compaction requested=false 2024-11-09T21:16:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:11,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-09T21:16:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-09T21:16:11,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-09T21:16:11,251 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3130 sec 2024-11-09T21:16:11,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.3260 sec 2024-11-09T21:16:11,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:11,305 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:16:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:11,306 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:11,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095620aa87d8e34867a8b9d1dd664a9347_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:11,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741965_1141 (size=14594) 2024-11-09T21:16:11,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187031388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,399 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187031400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,406 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187031507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,508 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187031518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,523 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187031716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,717 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,718 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:11,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187031727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:11,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:11,787 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:11,795 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095620aa87d8e34867a8b9d1dd664a9347_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095620aa87d8e34867a8b9d1dd664a9347_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:11,801 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/b64c8f248e3a4ebcad7435b816cd413f, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:11,801 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/b64c8f248e3a4ebcad7435b816cd413f is 175, key is test_row_0/A:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:11,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741966_1142 (size=39549) 2024-11-09T21:16:11,850 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=110, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/b64c8f248e3a4ebcad7435b816cd413f 2024-11-09T21:16:11,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/c5548bb0a3fd4495b15dd1c9af515f67 is 50, key is test_row_0/B:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:11,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741967_1143 (size=12001) 2024-11-09T21:16:11,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/c5548bb0a3fd4495b15dd1c9af515f67 2024-11-09T21:16:11,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/4c8e02ad587a40c5ad93d7e16fbfcf8f is 50, key is test_row_0/C:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:12,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741968_1144 (size=12001) 2024-11-09T21:16:12,012 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=110 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/4c8e02ad587a40c5ad93d7e16fbfcf8f 2024-11-09T21:16:12,029 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187032027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/b64c8f248e3a4ebcad7435b816cd413f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f 2024-11-09T21:16:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f, entries=200, sequenceid=110, filesize=38.6 K 2024-11-09T21:16:12,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/c5548bb0a3fd4495b15dd1c9af515f67 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c5548bb0a3fd4495b15dd1c9af515f67 2024-11-09T21:16:12,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c5548bb0a3fd4495b15dd1c9af515f67, entries=150, sequenceid=110, filesize=11.7 K 2024-11-09T21:16:12,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/4c8e02ad587a40c5ad93d7e16fbfcf8f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/4c8e02ad587a40c5ad93d7e16fbfcf8f 2024-11-09T21:16:12,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187032046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,065 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/4c8e02ad587a40c5ad93d7e16fbfcf8f, entries=150, sequenceid=110, filesize=11.7 K 2024-11-09T21:16:12,073 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aeea20555c2d70c5adac9bfee3363c0e in 768ms, sequenceid=110, compaction requested=true 2024-11-09T21:16:12,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:12,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:12,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:12,073 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:12,073 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:12,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:12,073 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:12,074 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:12,075 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:12,075 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:12,076 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:12,076 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:12,076 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,076 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,076 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/1afa62833cfe4817a300748ce89bcc39, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/235409bd313b44c9a764308e19e00665, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c5548bb0a3fd4495b15dd1c9af515f67] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=35.3 K 2024-11-09T21:16:12,076 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/fceba0c46f8a4218986a10acf5556aa7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=99.2 K 2024-11-09T21:16:12,076 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,076 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/fceba0c46f8a4218986a10acf5556aa7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f] 2024-11-09T21:16:12,077 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1afa62833cfe4817a300748ce89bcc39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186968566 2024-11-09T21:16:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-09T21:16:12,077 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting fceba0c46f8a4218986a10acf5556aa7, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186968566 2024-11-09T21:16:12,077 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:12,077 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting bfac464002ee431a8e7f4d73db6b6fbf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731186969900 2024-11-09T21:16:12,077 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 235409bd313b44c9a764308e19e00665, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731186969900 2024-11-09T21:16:12,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:12,078 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b64c8f248e3a4ebcad7435b816cd413f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731186971163 2024-11-09T21:16:12,078 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c5548bb0a3fd4495b15dd1c9af515f67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731186971180 2024-11-09T21:16:12,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-09T21:16:12,080 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:12,082 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:12,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:12,105 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:12,125 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:12,125 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/34c318a94e204a56814ea746c34c3f44 is 50, key is test_row_0/B:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:12,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110986e6cd4683f74249883a6d6f4951bcad_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:12,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110986e6cd4683f74249883a6d6f4951bcad_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:12,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110986e6cd4683f74249883a6d6f4951bcad_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:12,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:12,147 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T21:16:12,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186971376, max timestamp=1731186972144Num uniques 859; 2024-11-09T21:16:12,151 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:12,153 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186971376, max timestamp=1731186972144Num uniques 859; 2024-11-09T21:16:12,154 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:12,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731186971376, max timestamp=1731186972144Num uniques 859; 2024-11-09T21:16:12,157 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-09T21:16:12,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741969_1145 (size=12207) 2024-11-09T21:16:12,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741970_1146 (size=4469) 2024-11-09T21:16:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:12,161 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:12,162 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#123 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:12,164 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/c10fec656d61469089071aa7af503786 is 175, key is test_row_0/A:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:12,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:12,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095a567d01f688428d8e1eb7ab189f425b_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186971376/Put/seqid=0 2024-11-09T21:16:12,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741971_1147 (size=31161) 2024-11-09T21:16:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731186972169, max timestamp=1731186972223Num uniques 859; 2024-11-09T21:16:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731186972169, max timestamp=1731186972223Num uniques 859; 2024-11-09T21:16:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-09T21:16:12,226 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-09T21:16:12,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=409, cellSize=18725, totalHeapSize=42776, min timestamp=1731186972169, max timestamp=1731186972223Num uniques 859; 2024-11-09T21:16:12,229 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=18725, suffix heap size=117064,new segment heap size=42776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=409 2024-11-09T21:16:12,246 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,249 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/c10fec656d61469089071aa7af503786 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/c10fec656d61469089071aa7af503786 2024-11-09T21:16:12,259 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into c10fec656d61469089071aa7af503786(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:12,259 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:12,259 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186972073; duration=0sec 2024-11-09T21:16:12,260 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:12,260 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:12,260 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:12,261 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:12,261 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:12,261 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,262 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8118fb2d8345474784133c664284c45f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/bae48035fef94befac04361914e6b631, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/4c8e02ad587a40c5ad93d7e16fbfcf8f] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=35.3 K 2024-11-09T21:16:12,262 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8118fb2d8345474784133c664284c45f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186968566 2024-11-09T21:16:12,262 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting bae48035fef94befac04361914e6b631, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731186969900 2024-11-09T21:16:12,263 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c8e02ad587a40c5ad93d7e16fbfcf8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731186971180 2024-11-09T21:16:12,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741972_1148 (size=14744) 2024-11-09T21:16:12,298 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:12,299 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/388baee743674de99228b056d79b7965 is 50, key is test_row_0/C:col10/1731186971302/Put/seqid=0 2024-11-09T21:16:12,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187032308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187032311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,322 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187032314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,323 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741973_1149 (size=12207) 2024-11-09T21:16:12,371 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/388baee743674de99228b056d79b7965 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/388baee743674de99228b056d79b7965 2024-11-09T21:16:12,382 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 388baee743674de99228b056d79b7965(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:12,382 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:12,382 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186972073; duration=0sec 2024-11-09T21:16:12,382 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:12,382 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:12,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:12,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187032426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187032426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187032426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187032536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:12,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,565 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/34c318a94e204a56814ea746c34c3f44 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/34c318a94e204a56814ea746c34c3f44 2024-11-09T21:16:12,572 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into 34c318a94e204a56814ea746c34c3f44(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:12,572 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:12,572 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186972073; duration=0sec 2024-11-09T21:16:12,572 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:12,572 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:12,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187032576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187032637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187032637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187032637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,683 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:12,688 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095a567d01f688428d8e1eb7ab189f425b_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095a567d01f688428d8e1eb7ab189f425b_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:12,689 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/5ffa13e1ae1348c7b5daa0572fe36736, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:12,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/5ffa13e1ae1348c7b5daa0572fe36736 is 175, key is test_row_0/A:col10/1731186971376/Put/seqid=0 2024-11-09T21:16:12,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741974_1150 (size=39699) 2024-11-09T21:16:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:12,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:12,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:12,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,714 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:12,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:12,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:12,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187032946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187032946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:12,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187032957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,095 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/5ffa13e1ae1348c7b5daa0572fe36736 2024-11-09T21:16:13,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/9eaebb070cae47ca9d1a44f8754c3cb2 is 50, key is test_row_0/B:col10/1731186971376/Put/seqid=0 2024-11-09T21:16:13,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741975_1151 (size=12101) 2024-11-09T21:16:13,122 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=10.79 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/9eaebb070cae47ca9d1a44f8754c3cb2 2024-11-09T21:16:13,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/c7b1790e611349f187d907b686ca5753 is 50, key is test_row_0/C:col10/1731186971376/Put/seqid=0 2024-11-09T21:16:13,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741976_1152 (size=12101) 2024-11-09T21:16:13,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:13,358 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,358 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187033456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187033456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187033466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,543 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/c7b1790e611349f187d907b686ca5753 2024-11-09T21:16:13,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187033546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,549 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:13,549 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/5ffa13e1ae1348c7b5daa0572fe36736 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736 2024-11-09T21:16:13,554 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736, entries=200, sequenceid=132, filesize=38.8 K 2024-11-09T21:16:13,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/9eaebb070cae47ca9d1a44f8754c3cb2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/9eaebb070cae47ca9d1a44f8754c3cb2 2024-11-09T21:16:13,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/9eaebb070cae47ca9d1a44f8754c3cb2, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:16:13,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/c7b1790e611349f187d907b686ca5753 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c7b1790e611349f187d907b686ca5753 2024-11-09T21:16:13,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c7b1790e611349f187d907b686ca5753, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:16:13,567 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.15 KB/33945, heapSize ~85.20 KB/87240, currentSize=161.02 KB/164880 for aeea20555c2d70c5adac9bfee3363c0e in 1421ms, sequenceid=132, compaction requested=false 2024-11-09T21:16:13,567 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:13,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=167.72 KB heapSize=461.16 KB 2024-11-09T21:16:13,591 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:13,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:13,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:13,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:13,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:13,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:13,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109641d94c62e9348ee97e1293ee7b18bc0_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:13,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187033608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741977_1153 (size=14794) 2024-11-09T21:16:13,615 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:13,619 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109641d94c62e9348ee97e1293ee7b18bc0_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109641d94c62e9348ee97e1293ee7b18bc0_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:13,620 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/07d1ec0bc24347cda4848d375b699ca4, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:13,621 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/07d1ec0bc24347cda4848d375b699ca4 is 175, key is test_row_0/A:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:13,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741978_1154 (size=39749) 2024-11-09T21:16:13,664 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187033716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,719 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:13,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187033927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:13,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:13,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:13,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:13,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:13,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:13,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,032 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/07d1ec0bc24347cda4848d375b699ca4 2024-11-09T21:16:14,040 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/98139b1ffea6494d8a6a9d802dc7f18b is 50, key is test_row_0/B:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:14,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741979_1155 (size=12151) 2024-11-09T21:16:14,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:14,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:14,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187034246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:14,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:14,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:14,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,430 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,444 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/98139b1ffea6494d8a6a9d802dc7f18b 2024-11-09T21:16:14,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/c1373e10b11949938ce61caa153f54ce is 50, key is test_row_0/C:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:14,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T21:16:14,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T21:16:14,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741980_1156 (size=12151) 2024-11-09T21:16:14,472 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:14,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187034466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:14,473 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,473 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:14,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187034476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:14,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:14,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187034476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:14,477 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,477 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:14,583 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:14,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:14,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,583 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,736 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:14,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187034757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:14,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:14,862 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/c1373e10b11949938ce61caa153f54ce 2024-11-09T21:16:14,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/07d1ec0bc24347cda4848d375b699ca4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4 2024-11-09T21:16:14,880 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4, entries=200, sequenceid=172, filesize=38.8 K 2024-11-09T21:16:14,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/98139b1ffea6494d8a6a9d802dc7f18b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/98139b1ffea6494d8a6a9d802dc7f18b 2024-11-09T21:16:14,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/98139b1ffea6494d8a6a9d802dc7f18b, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T21:16:14,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/c1373e10b11949938ce61caa153f54ce as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c1373e10b11949938ce61caa153f54ce 2024-11-09T21:16:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c1373e10b11949938ce61caa153f54ce, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T21:16:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,899 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~461.11 KB/472176, currentSize=20.13 KB/20610 for aeea20555c2d70c5adac9bfee3363c0e in 1307ms, sequenceid=172, compaction requested=true 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:14,899 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:14,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:14,899 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:14,899 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:14,900 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110609 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:14,900 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:14,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:14,900 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,900 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,901 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/34c318a94e204a56814ea746c34c3f44, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/9eaebb070cae47ca9d1a44f8754c3cb2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/98139b1ffea6494d8a6a9d802dc7f18b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=35.6 K 2024-11-09T21:16:14,901 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/c10fec656d61469089071aa7af503786, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=108.0 K 2024-11-09T21:16:14,901 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:14,901 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/c10fec656d61469089071aa7af503786, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4] 2024-11-09T21:16:14,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,901 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 34c318a94e204a56814ea746c34c3f44, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731186971180 2024-11-09T21:16:14,901 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c10fec656d61469089071aa7af503786, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731186971180 2024-11-09T21:16:14,901 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eaebb070cae47ca9d1a44f8754c3cb2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731186971376 2024-11-09T21:16:14,902 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ffa13e1ae1348c7b5daa0572fe36736, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731186971376 2024-11-09T21:16:14,902 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 98139b1ffea6494d8a6a9d802dc7f18b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731186972283 2024-11-09T21:16:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,902 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07d1ec0bc24347cda4848d375b699ca4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731186972283 2024-11-09T21:16:14,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,931 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:14,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,933 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#132 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:14,933 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/cc23bc235fcf455eae83ab531000dc36 is 50, key is test_row_0/B:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,934 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109d04197cd325f438d83a556cc6e108ca0_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:14,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,936 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109d04197cd325f438d83a556cc6e108ca0_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:14,937 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d04197cd325f438d83a556cc6e108ca0_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741981_1157 (size=12459) 2024-11-09T21:16:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741982_1158 (size=4469) 2024-11-09T21:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,983 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#133 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:14,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,984 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/4a4659f879de4cc09f76e9e5c23b1319 is 175, key is test_row_0/A:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741983_1159 (size=31413) 2024-11-09T21:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,022 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/4a4659f879de4cc09f76e9e5c23b1319 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4a4659f879de4cc09f76e9e5c23b1319 2024-11-09T21:16:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,029 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into 4a4659f879de4cc09f76e9e5c23b1319(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:15,029 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:15,029 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186974899; duration=0sec 2024-11-09T21:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,029 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:15,029 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,029 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,030 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,031 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:15,031 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,031 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/388baee743674de99228b056d79b7965, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c7b1790e611349f187d907b686ca5753, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c1373e10b11949938ce61caa153f54ce] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=35.6 K 2024-11-09T21:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,031 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 388baee743674de99228b056d79b7965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=110, earliestPutTs=1731186971180 2024-11-09T21:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,032 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b1790e611349f187d907b686ca5753, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731186971376 2024-11-09T21:16:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,032 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1373e10b11949938ce61caa153f54ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731186972283 2024-11-09T21:16:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,042 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#134 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:15,043 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/b67770bd1fc544df99a57ae70a2ca1d4 is 50, key is test_row_0/C:col10/1731186972283/Put/seqid=0 2024-11-09T21:16:15,043 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-09T21:16:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:15,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741984_1160 (size=12459) 2024-11-09T21:16:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109be6f7da92f9b4e16b4813cdbbd5c1175_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_1/A:col10/1731186973605/Put/seqid=0 2024-11-09T21:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741985_1161 (size=7324) 2024-11-09T21:16:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,100 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109be6f7da92f9b4e16b4813cdbbd5c1175_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109be6f7da92f9b4e16b4813cdbbd5c1175_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/eb3cd2c0b9f54f6b8c32164a2546cdf1, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/eb3cd2c0b9f54f6b8c32164a2546cdf1 is 175, key is test_row_1/A:col10/1731186973605/Put/seqid=0 2024-11-09T21:16:15,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741986_1162 (size=13815) 2024-11-09T21:16:15,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,130 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=179, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/eb3cd2c0b9f54f6b8c32164a2546cdf1 2024-11-09T21:16:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/4db50baaaf2e43c789632efc0da45821 is 50, key is test_row_1/B:col10/1731186973605/Put/seqid=0 2024-11-09T21:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741987_1163 (size=7365) 2024-11-09T21:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/4db50baaaf2e43c789632efc0da45821 2024-11-09T21:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/67aa10afc6434afb822b1877dea13a23 is 50, key is test_row_1/C:col10/1731186973605/Put/seqid=0 2024-11-09T21:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741988_1164 (size=7365) 2024-11-09T21:16:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,164 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=179 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/67aa10afc6434afb822b1877dea13a23 2024-11-09T21:16:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/eb3cd2c0b9f54f6b8c32164a2546cdf1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1 2024-11-09T21:16:15,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1, entries=50, sequenceid=179, filesize=13.5 K 2024-11-09T21:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/4db50baaaf2e43c789632efc0da45821 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4db50baaaf2e43c789632efc0da45821 2024-11-09T21:16:15,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,183 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4db50baaaf2e43c789632efc0da45821, entries=50, sequenceid=179, filesize=7.2 K 2024-11-09T21:16:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/67aa10afc6434afb822b1877dea13a23 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/67aa10afc6434afb822b1877dea13a23 2024-11-09T21:16:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,191 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/67aa10afc6434afb822b1877dea13a23, entries=50, sequenceid=179, filesize=7.2 K 2024-11-09T21:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for aeea20555c2d70c5adac9bfee3363c0e in 149ms, sequenceid=179, compaction requested=false 2024-11-09T21:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:15,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-09T21:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-09T21:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-09T21:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,196 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1120 sec 2024-11-09T21:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 3.1190 sec 2024-11-09T21:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,361 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/cc23bc235fcf455eae83ab531000dc36 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/cc23bc235fcf455eae83ab531000dc36 2024-11-09T21:16:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,371 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into cc23bc235fcf455eae83ab531000dc36(size=12.2 K), total size for store is 19.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:15,371 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:15,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,372 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186974899; duration=0sec 2024-11-09T21:16:15,372 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:15,372 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,478 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/b67770bd1fc544df99a57ae70a2ca1d4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/b67770bd1fc544df99a57ae70a2ca1d4 2024-11-09T21:16:15,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,484 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into b67770bd1fc544df99a57ae70a2ca1d4(size=12.2 K), total size for store is 19.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:15,484 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:15,484 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186974899; duration=0sec 2024-11-09T21:16:15,484 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:15,484 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,695 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:16:15,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,695 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:15,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110965730fa65cc940a182fbfce16c65eb6f_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:15,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741989_1165 (size=22268) 2024-11-09T21:16:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,736 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,740 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110965730fa65cc940a182fbfce16c65eb6f_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110965730fa65cc940a182fbfce16c65eb6f_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:15,743 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/76742d992ec94f52ae4dfca83d1b2e66, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:15,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/76742d992ec94f52ae4dfca83d1b2e66 is 175, key is test_row_0/A:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:15,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:15,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741990_1166 (size=65673) 2024-11-09T21:16:15,774 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/76742d992ec94f52ae4dfca83d1b2e66 2024-11-09T21:16:15,789 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/de105a102cda4f41a65c46657fdba529 is 50, key is test_row_0/B:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:15,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741991_1167 (size=12151) 2024-11-09T21:16:15,810 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186975767, max timestamp=1731186975805Num uniques 859; 2024-11-09T21:16:15,811 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:16:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186975767, max timestamp=1731186975805Num uniques 859; 2024-11-09T21:16:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:16:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186975767, max timestamp=1731186975805Num uniques 859; 2024-11-09T21:16:15,812 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:16:15,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/de105a102cda4f41a65c46657fdba529 2024-11-09T21:16:15,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5eda425e01bd435abc68c01d5cefd311 is 50, key is test_row_0/C:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:15,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741992_1168 (size=12151) 2024-11-09T21:16:15,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:15,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187035865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,869 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:15,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:15,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187035868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:15,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187035976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:15,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187035976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:15,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187036186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,188 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187036186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-09T21:16:16,247 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:16,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-09T21:16:16,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T21:16:16,249 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:16,250 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:16,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:16,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5eda425e01bd435abc68c01d5cefd311 2024-11-09T21:16:16,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/76742d992ec94f52ae4dfca83d1b2e66 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66 2024-11-09T21:16:16,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66, entries=350, sequenceid=191, filesize=64.1 K 2024-11-09T21:16:16,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/de105a102cda4f41a65c46657fdba529 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/de105a102cda4f41a65c46657fdba529 2024-11-09T21:16:16,280 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/de105a102cda4f41a65c46657fdba529, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T21:16:16,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5eda425e01bd435abc68c01d5cefd311 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5eda425e01bd435abc68c01d5cefd311 2024-11-09T21:16:16,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5eda425e01bd435abc68c01d5cefd311, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T21:16:16,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for aeea20555c2d70c5adac9bfee3363c0e in 599ms, sequenceid=191, compaction requested=true 2024-11-09T21:16:16,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:16,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:16,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:16,294 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:16,294 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:16,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:16,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:16,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:16,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:16,295 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:16,295 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110901 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:16,295 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:16,295 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:16,295 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:16,295 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:16,296 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/cc23bc235fcf455eae83ab531000dc36, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4db50baaaf2e43c789632efc0da45821, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/de105a102cda4f41a65c46657fdba529] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=31.2 K 2024-11-09T21:16:16,296 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4a4659f879de4cc09f76e9e5c23b1319, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=108.3 K 2024-11-09T21:16:16,296 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:16,296 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4a4659f879de4cc09f76e9e5c23b1319, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66] 2024-11-09T21:16:16,296 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting cc23bc235fcf455eae83ab531000dc36, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731186972283 2024-11-09T21:16:16,296 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a4659f879de4cc09f76e9e5c23b1319, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731186972283 2024-11-09T21:16:16,297 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4db50baaaf2e43c789632efc0da45821, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731186973605 2024-11-09T21:16:16,297 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb3cd2c0b9f54f6b8c32164a2546cdf1, keycount=50, bloomtype=ROW, size=13.5 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731186973605 2024-11-09T21:16:16,297 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76742d992ec94f52ae4dfca83d1b2e66, keycount=350, bloomtype=ROW, size=64.1 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731186975610 2024-11-09T21:16:16,297 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting de105a102cda4f41a65c46657fdba529, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731186975678 2024-11-09T21:16:16,310 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#141 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:16,311 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/86fcffa2fdda46b4a2c1c81f099cfbd4 is 50, key is test_row_0/B:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:16,319 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:16,334 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110956ac096d7866471b89c8fdc73fff0c9b_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:16,337 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110956ac096d7866471b89c8fdc73fff0c9b_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:16,338 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110956ac096d7866471b89c8fdc73fff0c9b_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:16,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741993_1169 (size=12561) 2024-11-09T21:16:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T21:16:16,367 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/86fcffa2fdda46b4a2c1c81f099cfbd4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/86fcffa2fdda46b4a2c1c81f099cfbd4 2024-11-09T21:16:16,374 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into 86fcffa2fdda46b4a2c1c81f099cfbd4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:16,375 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:16,375 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186976294; duration=0sec 2024-11-09T21:16:16,375 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:16,375 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:16,375 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:16,376 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:16,377 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:16,377 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:16,377 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/b67770bd1fc544df99a57ae70a2ca1d4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/67aa10afc6434afb822b1877dea13a23, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5eda425e01bd435abc68c01d5cefd311] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=31.2 K 2024-11-09T21:16:16,378 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b67770bd1fc544df99a57ae70a2ca1d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731186972283 2024-11-09T21:16:16,379 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 67aa10afc6434afb822b1877dea13a23, keycount=50, bloomtype=ROW, size=7.2 K, encoding=NONE, compression=NONE, seqNum=179, earliestPutTs=1731186973605 2024-11-09T21:16:16,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741994_1170 (size=4469) 2024-11-09T21:16:16,381 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5eda425e01bd435abc68c01d5cefd311, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731186975678 2024-11-09T21:16:16,384 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#142 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:16,385 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/239c3c963aee453e83c6788bdc6ef9a9 is 175, key is test_row_0/A:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:16,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741995_1171 (size=31515) 2024-11-09T21:16:16,397 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/239c3c963aee453e83c6788bdc6ef9a9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/239c3c963aee453e83c6788bdc6ef9a9 2024-11-09T21:16:16,400 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:16,401 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/13210335ef7d43e9a9ca1fe1930f2f4d is 50, key is test_row_0/C:col10/1731186975686/Put/seqid=0 2024-11-09T21:16:16,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-09T21:16:16,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:16,402 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-09T21:16:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:16,410 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into 239c3c963aee453e83c6788bdc6ef9a9(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:16,410 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:16,410 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186976294; duration=0sec 2024-11-09T21:16:16,410 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:16,410 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:16,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741996_1172 (size=12561) 2024-11-09T21:16:16,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110925555c3a496a475bb7867278e6fd5b79_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186975866/Put/seqid=0 2024-11-09T21:16:16,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741997_1173 (size=12304) 2024-11-09T21:16:16,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:16,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:16,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187036502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187036503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187036508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,511 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187036509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,512 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187036510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T21:16:16,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187036616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187036616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,617 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187036616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187036617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187036617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187036826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187036827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187036827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187036827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187036827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:16,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:16,839 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/13210335ef7d43e9a9ca1fe1930f2f4d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/13210335ef7d43e9a9ca1fe1930f2f4d 2024-11-09T21:16:16,847 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 13210335ef7d43e9a9ca1fe1930f2f4d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:16,847 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:16,848 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186976294; duration=0sec 2024-11-09T21:16:16,848 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:16,848 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:16,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:16,863 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110925555c3a496a475bb7867278e6fd5b79_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110925555c3a496a475bb7867278e6fd5b79_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:16,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/183dd99d0bcc48428aa5bd5797613eec, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:16,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/183dd99d0bcc48428aa5bd5797613eec is 175, key is test_row_0/A:col10/1731186975866/Put/seqid=0 2024-11-09T21:16:16,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T21:16:16,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741998_1174 (size=31105) 2024-11-09T21:16:16,898 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/183dd99d0bcc48428aa5bd5797613eec 2024-11-09T21:16:16,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/f0c2e4b7408e4d99859118e0f0f9e6cf is 50, key is test_row_0/B:col10/1731186975866/Put/seqid=0 2024-11-09T21:16:16,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741999_1175 (size=12151) 2024-11-09T21:16:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187037136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187037136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187037136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187037136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187037146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/f0c2e4b7408e4d99859118e0f0f9e6cf 2024-11-09T21:16:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/2f4295b507ca4d7f9e3bea3a6564a685 is 50, key is test_row_0/C:col10/1731186975866/Put/seqid=0 2024-11-09T21:16:17,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742000_1176 (size=12151) 2024-11-09T21:16:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T21:16:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187037647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187037647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187037647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187037656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187037656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:17,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:17,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/2f4295b507ca4d7f9e3bea3a6564a685 2024-11-09T21:16:17,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/183dd99d0bcc48428aa5bd5797613eec as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec 2024-11-09T21:16:17,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec, entries=150, sequenceid=227, filesize=30.4 K 2024-11-09T21:16:17,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/f0c2e4b7408e4d99859118e0f0f9e6cf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/f0c2e4b7408e4d99859118e0f0f9e6cf 2024-11-09T21:16:17,791 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/f0c2e4b7408e4d99859118e0f0f9e6cf, entries=150, sequenceid=227, filesize=11.9 K 2024-11-09T21:16:17,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/2f4295b507ca4d7f9e3bea3a6564a685 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/2f4295b507ca4d7f9e3bea3a6564a685 2024-11-09T21:16:17,798 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/2f4295b507ca4d7f9e3bea3a6564a685, entries=150, sequenceid=227, filesize=11.9 K 2024-11-09T21:16:17,799 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for aeea20555c2d70c5adac9bfee3363c0e in 1397ms, sequenceid=227, compaction requested=false 2024-11-09T21:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:17,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-09T21:16:17,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-09T21:16:17,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-09T21:16:17,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5510 sec 2024-11-09T21:16:17,807 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.5570 sec 2024-11-09T21:16:18,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-09T21:16:18,397 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:18,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:18,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-09T21:16:18,400 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:18,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T21:16:18,408 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:18,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:18,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T21:16:18,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-09T21:16:18,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:18,563 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:16:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:18,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:18,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097c41567aaa9b4212aeb7ee4a7233a4b5_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186976508/Put/seqid=0 2024-11-09T21:16:18,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742001_1177 (size=12304) 2024-11-09T21:16:18,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:18,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:18,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187038701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187038703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,712 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187038704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186978689, max timestamp=1731186978709Num uniques 859; 2024-11-09T21:16:18,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186978689, max timestamp=1731186978709Num uniques 859; 2024-11-09T21:16:18,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:16:18,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:16:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731186978689, max timestamp=1731186978709Num uniques 859; 2024-11-09T21:16:18,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:16:18,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T21:16:18,766 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187038758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187038767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,774 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187038832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187038832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187038833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187038876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:18,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187038877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:18,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:19,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097c41567aaa9b4212aeb7ee4a7233a4b5_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097c41567aaa9b4212aeb7ee4a7233a4b5_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:19,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0b1b9f552157490487141d71af0e2198, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0b1b9f552157490487141d71af0e2198 is 175, key is test_row_0/A:col10/1731186976508/Put/seqid=0 2024-11-09T21:16:19,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742002_1178 (size=31105) 2024-11-09T21:16:19,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T21:16:19,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187039036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,040 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187039046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187039046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187039086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187039086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187039346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,350 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187039356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187039356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,361 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,386 INFO [master/9514d6d5e5be:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-09T21:16:19,386 INFO [master/9514d6d5e5be:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-09T21:16:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187039396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187039396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,399 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,421 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0b1b9f552157490487141d71af0e2198 2024-11-09T21:16:19,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/5cd7baffb1114fe8bf2f3e1c5777b2d4 is 50, key is test_row_0/B:col10/1731186976508/Put/seqid=0 2024-11-09T21:16:19,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742003_1179 (size=12151) 2024-11-09T21:16:19,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T21:16:19,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/5cd7baffb1114fe8bf2f3e1c5777b2d4 2024-11-09T21:16:19,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/09442dce1d3b41888ea5d039ada18829 is 50, key is test_row_0/C:col10/1731186976508/Put/seqid=0 2024-11-09T21:16:19,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187039859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,860 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742004_1180 (size=12151) 2024-11-09T21:16:19,868 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/09442dce1d3b41888ea5d039ada18829 2024-11-09T21:16:19,872 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187039867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,873 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0b1b9f552157490487141d71af0e2198 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198 2024-11-09T21:16:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187039877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,891 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198, entries=150, sequenceid=240, filesize=30.4 K 2024-11-09T21:16:19,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/5cd7baffb1114fe8bf2f3e1c5777b2d4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5cd7baffb1114fe8bf2f3e1c5777b2d4 2024-11-09T21:16:19,899 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5cd7baffb1114fe8bf2f3e1c5777b2d4, entries=150, sequenceid=240, filesize=11.9 K 2024-11-09T21:16:19,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/09442dce1d3b41888ea5d039ada18829 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09442dce1d3b41888ea5d039ada18829 2024-11-09T21:16:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187039906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187039906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:19,911 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09442dce1d3b41888ea5d039ada18829, entries=150, sequenceid=240, filesize=11.9 K 2024-11-09T21:16:19,914 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for aeea20555c2d70c5adac9bfee3363c0e in 1351ms, sequenceid=240, compaction requested=true 2024-11-09T21:16:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:19,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-09T21:16:19,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-09T21:16:19,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-09T21:16:19,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-11-09T21:16:19,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.5210 sec 2024-11-09T21:16:20,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-09T21:16:20,556 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:20,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-09T21:16:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T21:16:20,559 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:20,559 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:20,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:20,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T21:16:20,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:20,712 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:20,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:20,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fc311a39268b4bec8bc70c7cb3fa6097_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186978756/Put/seqid=0 2024-11-09T21:16:20,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742005_1181 (size=12454) 2024-11-09T21:16:20,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:20,756 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fc311a39268b4bec8bc70c7cb3fa6097_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fc311a39268b4bec8bc70c7cb3fa6097_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/8dd25477a79642d9be3f2144216c8e43, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:20,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/8dd25477a79642d9be3f2144216c8e43 is 175, key is test_row_0/A:col10/1731186978756/Put/seqid=0 2024-11-09T21:16:20,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742006_1182 (size=31255) 2024-11-09T21:16:20,805 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/8dd25477a79642d9be3f2144216c8e43 2024-11-09T21:16:20,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/b211e934c6a44dbf93ed0c20734a2c9b is 50, key is test_row_0/B:col10/1731186978756/Put/seqid=0 2024-11-09T21:16:20,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742007_1183 (size=12301) 2024-11-09T21:16:20,857 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/b211e934c6a44dbf93ed0c20734a2c9b 2024-11-09T21:16:20,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/f3303cd4e9da4191a7be3263754e639e is 50, key is test_row_0/C:col10/1731186978756/Put/seqid=0 2024-11-09T21:16:20,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T21:16:20,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:20,880 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:20,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742008_1184 (size=12301) 2024-11-09T21:16:20,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:20,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187040900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,902 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,903 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:20,908 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:20,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187040902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,909 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:20,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187040902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,911 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:20,920 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187040917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:20,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:20,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187040917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:20,921 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,922 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:20,923 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:20,923 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:20,924 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187041006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187041016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187041016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,021 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T21:16:21,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187041216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187041226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187041226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,299 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/f3303cd4e9da4191a7be3263754e639e 2024-11-09T21:16:21,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/8dd25477a79642d9be3f2144216c8e43 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43 2024-11-09T21:16:21,317 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43, entries=150, sequenceid=274, filesize=30.5 K 2024-11-09T21:16:21,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/b211e934c6a44dbf93ed0c20734a2c9b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b211e934c6a44dbf93ed0c20734a2c9b 2024-11-09T21:16:21,323 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b211e934c6a44dbf93ed0c20734a2c9b, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T21:16:21,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/f3303cd4e9da4191a7be3263754e639e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/f3303cd4e9da4191a7be3263754e639e 2024-11-09T21:16:21,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/f3303cd4e9da4191a7be3263754e639e, entries=150, sequenceid=274, filesize=12.0 K 2024-11-09T21:16:21,331 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=53.67 KB/54960 for aeea20555c2d70c5adac9bfee3363c0e in 619ms, sequenceid=274, compaction requested=true 2024-11-09T21:16:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-09T21:16:21,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-09T21:16:21,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-09T21:16:21,334 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 773 msec 2024-11-09T21:16:21,337 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 778 msec 2024-11-09T21:16:21,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:21,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:16:21,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:21,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:21,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:21,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:21,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:21,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:21,563 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110910013437f50f4f7e8162147f394b4ba7_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:21,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742009_1185 (size=14994) 2024-11-09T21:16:21,581 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187041577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,581 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,584 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187041578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,584 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187041582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,590 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187041686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187041686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,687 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-09T21:16:21,697 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:21,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:21,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-09T21:16:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:21,699 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:21,700 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:21,700 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:21,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187041696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:21,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:21,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:21,852 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:21,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:21,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187041896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187041896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,906 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,907 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:21,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187041907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:21,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:21,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:21,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110910013437f50f4f7e8162147f394b4ba7_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110910013437f50f4f7e8162147f394b4ba7_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:21,982 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/96b8c9a3dac343859224012bf52fff4a, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:21,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/96b8c9a3dac343859224012bf52fff4a is 175, key is test_row_0/A:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:21,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742010_1186 (size=39949) 2024-11-09T21:16:22,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:22,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,220 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187042216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187042216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187042216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,313 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,313 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:22,388 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/96b8c9a3dac343859224012bf52fff4a 2024-11-09T21:16:22,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/90723ee9f98d4851a4b21adb46f54acb is 50, key is test_row_0/B:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:22,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742011_1187 (size=12301) 2024-11-09T21:16:22,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187042726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187042737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187042737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,738 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,804 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/90723ee9f98d4851a4b21adb46f54acb 2024-11-09T21:16:22,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/7fa46f38383443808dd0f7c736abe351 is 50, key is test_row_0/C:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:22,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742012_1188 (size=12301) 2024-11-09T21:16:22,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:22,925 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:22,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:22,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187042937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,938 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:22,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:22,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187042946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:22,952 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:23,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:23,229 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/7fa46f38383443808dd0f7c736abe351 2024-11-09T21:16:23,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:23,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:23,231 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,231 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:23,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:23,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:23,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/96b8c9a3dac343859224012bf52fff4a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a 2024-11-09T21:16:23,248 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a, entries=200, sequenceid=286, filesize=39.0 K 2024-11-09T21:16:23,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/90723ee9f98d4851a4b21adb46f54acb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/90723ee9f98d4851a4b21adb46f54acb 2024-11-09T21:16:23,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/90723ee9f98d4851a4b21adb46f54acb, entries=150, sequenceid=286, filesize=12.0 K 2024-11-09T21:16:23,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/7fa46f38383443808dd0f7c736abe351 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/7fa46f38383443808dd0f7c736abe351 2024-11-09T21:16:23,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/7fa46f38383443808dd0f7c736abe351, entries=150, sequenceid=286, filesize=12.0 K 2024-11-09T21:16:23,263 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for aeea20555c2d70c5adac9bfee3363c0e in 1731ms, sequenceid=286, compaction requested=true 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:23,263 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T21:16:23,263 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:23,263 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:23,265 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164929 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T21:16:23,265 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61465 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T21:16:23,265 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:23,265 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:23,265 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,265 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,265 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/86fcffa2fdda46b4a2c1c81f099cfbd4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/f0c2e4b7408e4d99859118e0f0f9e6cf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5cd7baffb1114fe8bf2f3e1c5777b2d4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b211e934c6a44dbf93ed0c20734a2c9b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/90723ee9f98d4851a4b21adb46f54acb] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=60.0 K 2024-11-09T21:16:23,265 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/239c3c963aee453e83c6788bdc6ef9a9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=161.1 K 2024-11-09T21:16:23,265 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,265 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/239c3c963aee453e83c6788bdc6ef9a9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a] 2024-11-09T21:16:23,266 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 86fcffa2fdda46b4a2c1c81f099cfbd4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731186975678 2024-11-09T21:16:23,266 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f0c2e4b7408e4d99859118e0f0f9e6cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731186975856 2024-11-09T21:16:23,266 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 239c3c963aee453e83c6788bdc6ef9a9, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731186975678 2024-11-09T21:16:23,266 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 183dd99d0bcc48428aa5bd5797613eec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731186975856 2024-11-09T21:16:23,267 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cd7baffb1114fe8bf2f3e1c5777b2d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731186976507 2024-11-09T21:16:23,267 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b1b9f552157490487141d71af0e2198, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731186976507 2024-11-09T21:16:23,267 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b211e934c6a44dbf93ed0c20734a2c9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731186978752 2024-11-09T21:16:23,267 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8dd25477a79642d9be3f2144216c8e43, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731186978752 2024-11-09T21:16:23,267 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 90723ee9f98d4851a4b21adb46f54acb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731186980898 2024-11-09T21:16:23,267 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b8c9a3dac343859224012bf52fff4a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731186980898 2024-11-09T21:16:23,280 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#156 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:23,281 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/dea9b50947b24d1fa7e1205b69bb83b0 is 50, key is test_row_0/B:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:23,282 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:23,284 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109b889038170304863beb0921954bc9957_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:23,286 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109b889038170304863beb0921954bc9957_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:23,287 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b889038170304863beb0921954bc9957_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:23,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742013_1189 (size=12881) 2024-11-09T21:16:23,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742014_1190 (size=4469) 2024-11-09T21:16:23,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-09T21:16:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:16:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:23,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:23,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d09fd5cfbd7e463bab3f3df498968e9a_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186981575/Put/seqid=0 2024-11-09T21:16:23,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742015_1191 (size=12454) 2024-11-09T21:16:23,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:23,425 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109d09fd5cfbd7e463bab3f3df498968e9a_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d09fd5cfbd7e463bab3f3df498968e9a_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:23,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/3dc2f487d8204a59a744c9bf57118129, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:23,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/3dc2f487d8204a59a744c9bf57118129 is 175, key is test_row_0/A:col10/1731186981575/Put/seqid=0 2024-11-09T21:16:23,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742016_1192 (size=31255) 2024-11-09T21:16:23,696 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/dea9b50947b24d1fa7e1205b69bb83b0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/dea9b50947b24d1fa7e1205b69bb83b0 2024-11-09T21:16:23,700 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#157 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:23,701 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/cb83607ee22446e79b2a862a8947f0c0 is 175, key is test_row_0/A:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:23,706 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into dea9b50947b24d1fa7e1205b69bb83b0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:23,706 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:23,706 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=11, startTime=1731186983263; duration=0sec 2024-11-09T21:16:23,706 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:23,706 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:23,706 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T21:16:23,709 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61465 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T21:16:23,709 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:23,710 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:23,710 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/13210335ef7d43e9a9ca1fe1930f2f4d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/2f4295b507ca4d7f9e3bea3a6564a685, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09442dce1d3b41888ea5d039ada18829, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/f3303cd4e9da4191a7be3263754e639e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/7fa46f38383443808dd0f7c736abe351] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=60.0 K 2024-11-09T21:16:23,711 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 13210335ef7d43e9a9ca1fe1930f2f4d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731186975678 2024-11-09T21:16:23,711 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f4295b507ca4d7f9e3bea3a6564a685, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731186975856 2024-11-09T21:16:23,712 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 09442dce1d3b41888ea5d039ada18829, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731186976507 2024-11-09T21:16:23,712 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f3303cd4e9da4191a7be3263754e639e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731186978752 2024-11-09T21:16:23,713 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 7fa46f38383443808dd0f7c736abe351, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731186980898 2024-11-09T21:16:23,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742017_1193 (size=31835) 2024-11-09T21:16:23,734 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#159 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:23,734 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/0971bad10544453888417af3d652074c is 50, key is test_row_0/C:col10/1731186981529/Put/seqid=0 2024-11-09T21:16:23,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:23,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:23,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742018_1194 (size=12881) 2024-11-09T21:16:23,769 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/0971bad10544453888417af3d652074c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/0971bad10544453888417af3d652074c 2024-11-09T21:16:23,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187043766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:23,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:23,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187043768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,775 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:23,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187043772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,779 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 0971bad10544453888417af3d652074c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:23,779 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:23,779 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=11, startTime=1731186983263; duration=0sec 2024-11-09T21:16:23,780 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:23,780 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:23,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/3dc2f487d8204a59a744c9bf57118129 2024-11-09T21:16:23,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:23,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/188dd6e5ddc340188af987fc7527cd47 is 50, key is test_row_0/B:col10/1731186981575/Put/seqid=0 2024-11-09T21:16:23,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742019_1195 (size=12301) 2024-11-09T21:16:23,874 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/188dd6e5ddc340188af987fc7527cd47 2024-11-09T21:16:23,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187043887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:23,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187043887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187043887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:23,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:23,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5bcecb79695249b5bbaf52f8be28b1ed is 50, key is test_row_0/C:col10/1731186981575/Put/seqid=0 2024-11-09T21:16:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742020_1196 (size=12301) 2024-11-09T21:16:24,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187044097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,097 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,097 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187044097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187044097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,138 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/cb83607ee22446e79b2a862a8947f0c0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cb83607ee22446e79b2a862a8947f0c0 2024-11-09T21:16:24,143 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into cb83607ee22446e79b2a862a8947f0c0(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:24,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:24,143 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=11, startTime=1731186983263; duration=0sec 2024-11-09T21:16:24,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:24,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:24,296 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5bcecb79695249b5bbaf52f8be28b1ed 2024-11-09T21:16:24,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/3dc2f487d8204a59a744c9bf57118129 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129 2024-11-09T21:16:24,307 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129, entries=150, sequenceid=306, filesize=30.5 K 2024-11-09T21:16:24,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/188dd6e5ddc340188af987fc7527cd47 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/188dd6e5ddc340188af987fc7527cd47 2024-11-09T21:16:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,312 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/188dd6e5ddc340188af987fc7527cd47, entries=150, sequenceid=306, filesize=12.0 K 2024-11-09T21:16:24,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/5bcecb79695249b5bbaf52f8be28b1ed as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5bcecb79695249b5bbaf52f8be28b1ed 2024-11-09T21:16:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,324 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5bcecb79695249b5bbaf52f8be28b1ed, entries=150, sequenceid=306, filesize=12.0 K 2024-11-09T21:16:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aeea20555c2d70c5adac9bfee3363c0e in 940ms, sequenceid=306, compaction requested=false 2024-11-09T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:24,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:24,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-09T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-09T21:16:24,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-09T21:16:24,328 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6260 sec 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.6320 sec 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:24,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,425 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:16:24,425 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:24,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:24,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:24,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:24,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:24,426 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109637ec770b27d41e58809fc057d2f49b1_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742021_1197 (size=12454) 2024-11-09T21:16:24,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187044482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187044497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,506 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187044501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,506 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187044597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187044616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187044616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187044806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,807 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187044826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187044826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:24,828 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:24,851 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:24,855 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109637ec770b27d41e58809fc057d2f49b1_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109637ec770b27d41e58809fc057d2f49b1_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:24,857 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f31a7ef86bb4450296d1d47c62f5bfcc, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:24,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f31a7ef86bb4450296d1d47c62f5bfcc is 175, key is test_row_0/A:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:24,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742022_1198 (size=31251) 2024-11-09T21:16:25,117 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:25,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187045116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:25,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187045137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:25,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187045136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:25,142 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:25,266 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=322, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f31a7ef86bb4450296d1d47c62f5bfcc 2024-11-09T21:16:25,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/fa71abc496ff40ad900fb291b5e60c04 is 50, key is test_row_0/B:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:25,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742023_1199 (size=9857) 2024-11-09T21:16:25,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187045626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:25,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187045646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:25,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187045656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:25,659 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:25,677 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/fa71abc496ff40ad900fb291b5e60c04 2024-11-09T21:16:25,685 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/832bcec4dac849dd9d403ea15ee60275 is 50, key is test_row_0/C:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:25,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742024_1200 (size=9857) 2024-11-09T21:16:25,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/832bcec4dac849dd9d403ea15ee60275 2024-11-09T21:16:25,694 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f31a7ef86bb4450296d1d47c62f5bfcc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc 2024-11-09T21:16:25,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc, entries=150, sequenceid=322, filesize=30.5 K 2024-11-09T21:16:25,707 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/fa71abc496ff40ad900fb291b5e60c04 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/fa71abc496ff40ad900fb291b5e60c04 2024-11-09T21:16:25,712 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/fa71abc496ff40ad900fb291b5e60c04, entries=100, sequenceid=322, filesize=9.6 K 2024-11-09T21:16:25,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/832bcec4dac849dd9d403ea15ee60275 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/832bcec4dac849dd9d403ea15ee60275 2024-11-09T21:16:25,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/832bcec4dac849dd9d403ea15ee60275, entries=100, sequenceid=322, filesize=9.6 K 2024-11-09T21:16:25,719 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for aeea20555c2d70c5adac9bfee3363c0e in 1294ms, sequenceid=322, compaction requested=true 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:25,719 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:25,719 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:25,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94341 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:25,721 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:25,721 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:25,721 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cb83607ee22446e79b2a862a8947f0c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=92.1 K 2024-11-09T21:16:25,721 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/dea9b50947b24d1fa7e1205b69bb83b0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/188dd6e5ddc340188af987fc7527cd47, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/fa71abc496ff40ad900fb291b5e60c04] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=34.2 K 2024-11-09T21:16:25,721 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cb83607ee22446e79b2a862a8947f0c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc] 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting dea9b50947b24d1fa7e1205b69bb83b0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731186980898 2024-11-09T21:16:25,721 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb83607ee22446e79b2a862a8947f0c0, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731186980898 2024-11-09T21:16:25,722 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3dc2f487d8204a59a744c9bf57118129, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731186981574 2024-11-09T21:16:25,722 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 188dd6e5ddc340188af987fc7527cd47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731186981574 2024-11-09T21:16:25,722 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f31a7ef86bb4450296d1d47c62f5bfcc, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731186983761 2024-11-09T21:16:25,722 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fa71abc496ff40ad900fb291b5e60c04, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731186983767 2024-11-09T21:16:25,734 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#165 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:25,734 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/515febcbda5c499796310441b9362196 is 50, key is test_row_0/B:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:25,736 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742025_1201 (size=12983) 2024-11-09T21:16:25,753 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109e5fdd1d5fe3d4b8b918140d238adf394_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:25,754 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109e5fdd1d5fe3d4b8b918140d238adf394_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:25,754 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e5fdd1d5fe3d4b8b918140d238adf394_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:25,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742026_1202 (size=4469) 2024-11-09T21:16:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-09T21:16:25,856 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:25,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-09T21:16:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T21:16:25,865 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:25,866 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:25,866 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T21:16:26,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-09T21:16:26,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:26,019 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:16:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:26,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:26,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099260c6ca9ab349d480b057ec364f2970_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186984485/Put/seqid=0 2024-11-09T21:16:26,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742027_1203 (size=12454) 2024-11-09T21:16:26,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:26,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099260c6ca9ab349d480b057ec364f2970_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099260c6ca9ab349d480b057ec364f2970_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/cfde799599c84d11a7281fbb2843db0a, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:26,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/cfde799599c84d11a7281fbb2843db0a is 175, key is test_row_0/A:col10/1731186984485/Put/seqid=0 2024-11-09T21:16:26,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742028_1204 (size=31255) 2024-11-09T21:16:26,043 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/cfde799599c84d11a7281fbb2843db0a 2024-11-09T21:16:26,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/14e915a706fd454aa52a936994f43ad3 is 50, key is test_row_0/B:col10/1731186984485/Put/seqid=0 2024-11-09T21:16:26,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742029_1205 (size=12301) 2024-11-09T21:16:26,155 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/515febcbda5c499796310441b9362196 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/515febcbda5c499796310441b9362196 2024-11-09T21:16:26,160 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into 515febcbda5c499796310441b9362196(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:26,160 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:26,160 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186985719; duration=0sec 2024-11-09T21:16:26,160 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#166 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:26,160 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:26,160 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:26,160 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:26,160 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/8cde54a816b048b5aa226fce3c7318bd is 175, key is test_row_0/A:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:26,162 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35039 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:26,162 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:26,162 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:26,162 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/0971bad10544453888417af3d652074c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5bcecb79695249b5bbaf52f8be28b1ed, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/832bcec4dac849dd9d403ea15ee60275] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=34.2 K 2024-11-09T21:16:26,162 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0971bad10544453888417af3d652074c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731186980898 2024-11-09T21:16:26,163 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bcecb79695249b5bbaf52f8be28b1ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731186981574 2024-11-09T21:16:26,163 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 832bcec4dac849dd9d403ea15ee60275, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731186983767 2024-11-09T21:16:26,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742030_1206 (size=32044) 2024-11-09T21:16:26,169 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/8cde54a816b048b5aa226fce3c7318bd as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8cde54a816b048b5aa226fce3c7318bd 2024-11-09T21:16:26,171 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#169 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:26,172 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/91f9db7c0b1e47fb8eddec79a360a8cb is 50, key is test_row_0/C:col10/1731186983767/Put/seqid=0 2024-11-09T21:16:26,174 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into 8cde54a816b048b5aa226fce3c7318bd(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:26,174 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:26,174 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186985719; duration=0sec 2024-11-09T21:16:26,174 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:26,174 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T21:16:26,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742031_1207 (size=12983) 2024-11-09T21:16:26,482 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/14e915a706fd454aa52a936994f43ad3 2024-11-09T21:16:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T21:16:26,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/fcccbceec66e4c76846da65c0b23a410 is 50, key is test_row_0/C:col10/1731186984485/Put/seqid=0 2024-11-09T21:16:26,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742032_1208 (size=12301) 2024-11-09T21:16:26,584 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/91f9db7c0b1e47fb8eddec79a360a8cb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/91f9db7c0b1e47fb8eddec79a360a8cb 2024-11-09T21:16:26,590 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 91f9db7c0b1e47fb8eddec79a360a8cb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:26,590 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:26,590 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186985719; duration=0sec 2024-11-09T21:16:26,590 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:26,590 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:26,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:26,638 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:26,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:26,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187046656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:26,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:26,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187046659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:26,660 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:26,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:26,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:26,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187046666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:26,672 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:26,672 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,673 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:26,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:26,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187046766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:26,769 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:26,909 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/fcccbceec66e4c76846da65c0b23a410 2024-11-09T21:16:26,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/cfde799599c84d11a7281fbb2843db0a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a 2024-11-09T21:16:26,921 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a, entries=150, sequenceid=342, filesize=30.5 K 2024-11-09T21:16:26,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/14e915a706fd454aa52a936994f43ad3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/14e915a706fd454aa52a936994f43ad3 2024-11-09T21:16:26,928 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/14e915a706fd454aa52a936994f43ad3, entries=150, sequenceid=342, filesize=12.0 K 2024-11-09T21:16:26,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/fcccbceec66e4c76846da65c0b23a410 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/fcccbceec66e4c76846da65c0b23a410 2024-11-09T21:16:26,939 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/fcccbceec66e4c76846da65c0b23a410, entries=150, sequenceid=342, filesize=12.0 K 2024-11-09T21:16:26,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for aeea20555c2d70c5adac9bfee3363c0e in 921ms, sequenceid=342, compaction requested=false 2024-11-09T21:16:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:26,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-09T21:16:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-09T21:16:26,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-09T21:16:26,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0760 sec 2024-11-09T21:16:26,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.0880 sec 2024-11-09T21:16:26,979 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:16:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:26,979 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:26,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:26,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c49bec34a23e4e9894e318c93ba41d0f_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-09T21:16:26,997 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:26,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-09T21:16:27,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:27,003 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:27,003 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T21:16:27,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187047019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742033_1209 (size=14994) 2024-11-09T21:16:27,030 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:27,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187047025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187047026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c49bec34a23e4e9894e318c93ba41d0f_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c49bec34a23e4e9894e318c93ba41d0f_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:27,038 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0fa998f9814e41aaa0751ec26801bc1d, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:27,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0fa998f9814e41aaa0751ec26801bc1d is 175, key is test_row_0/A:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:27,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742034_1210 (size=39949) 2024-11-09T21:16:27,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=358, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0fa998f9814e41aaa0751ec26801bc1d 2024-11-09T21:16:27,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/3982256cbc254dea8fa2ed3b11f28db5 is 50, key is test_row_0/B:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:27,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742035_1211 (size=12301) 2024-11-09T21:16:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T21:16:27,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187047136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,140 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187047146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187047146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,148 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,158 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-09T21:16:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-09T21:16:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T21:16:27,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187047346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187047356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187047356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,358 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-09T21:16:27,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:27,487 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/3982256cbc254dea8fa2ed3b11f28db5 2024-11-09T21:16:27,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/57c4b937574143db9f1da681dc6e0653 is 50, key is test_row_0/C:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:27,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742036_1212 (size=12301) 2024-11-09T21:16:27,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=358 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/57c4b937574143db9f1da681dc6e0653 2024-11-09T21:16:27,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0fa998f9814e41aaa0751ec26801bc1d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d 2024-11-09T21:16:27,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d, entries=200, sequenceid=358, filesize=39.0 K 2024-11-09T21:16:27,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/3982256cbc254dea8fa2ed3b11f28db5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3982256cbc254dea8fa2ed3b11f28db5 2024-11-09T21:16:27,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3982256cbc254dea8fa2ed3b11f28db5, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T21:16:27,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/57c4b937574143db9f1da681dc6e0653 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/57c4b937574143db9f1da681dc6e0653 2024-11-09T21:16:27,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/57c4b937574143db9f1da681dc6e0653, entries=150, sequenceid=358, filesize=12.0 K 2024-11-09T21:16:27,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for aeea20555c2d70c5adac9bfee3363c0e in 555ms, sequenceid=358, compaction requested=true 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:27,534 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:27,534 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:27,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:27,535 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103248 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:27,535 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:27,535 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:27,535 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:27,535 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,535 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,535 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8cde54a816b048b5aa226fce3c7318bd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=100.8 K 2024-11-09T21:16:27,535 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/515febcbda5c499796310441b9362196, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/14e915a706fd454aa52a936994f43ad3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3982256cbc254dea8fa2ed3b11f28db5] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=36.7 K 2024-11-09T21:16:27,535 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,535 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8cde54a816b048b5aa226fce3c7318bd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d] 2024-11-09T21:16:27,536 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8cde54a816b048b5aa226fce3c7318bd, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731186981574 2024-11-09T21:16:27,536 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 515febcbda5c499796310441b9362196, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731186981574 2024-11-09T21:16:27,537 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfde799599c84d11a7281fbb2843db0a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731186984465 2024-11-09T21:16:27,537 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 14e915a706fd454aa52a936994f43ad3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731186984465 2024-11-09T21:16:27,537 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3982256cbc254dea8fa2ed3b11f28db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731186986655 2024-11-09T21:16:27,537 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fa998f9814e41aaa0751ec26801bc1d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731186986650 2024-11-09T21:16:27,545 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:27,548 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110926acdea4144d4e82aef922f29c1803ad_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:27,548 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:27,549 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/565d2bddaf3445cfaa62f108b3699733 is 50, key is test_row_0/B:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:27,550 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110926acdea4144d4e82aef922f29c1803ad_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:27,550 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110926acdea4144d4e82aef922f29c1803ad_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:27,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742037_1213 (size=13085) 2024-11-09T21:16:27,580 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/565d2bddaf3445cfaa62f108b3699733 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/565d2bddaf3445cfaa62f108b3699733 2024-11-09T21:16:27,586 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into 565d2bddaf3445cfaa62f108b3699733(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:27,586 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:27,586 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186987534; duration=0sec 2024-11-09T21:16:27,586 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:27,586 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:27,586 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:27,588 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:27,588 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:27,588 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,588 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/91f9db7c0b1e47fb8eddec79a360a8cb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/fcccbceec66e4c76846da65c0b23a410, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/57c4b937574143db9f1da681dc6e0653] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=36.7 K 2024-11-09T21:16:27,589 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 91f9db7c0b1e47fb8eddec79a360a8cb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731186981574 2024-11-09T21:16:27,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742038_1214 (size=4469) 2024-11-09T21:16:27,589 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fcccbceec66e4c76846da65c0b23a410, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731186984465 2024-11-09T21:16:27,590 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 57c4b937574143db9f1da681dc6e0653, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731186986655 2024-11-09T21:16:27,602 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:27,603 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/9e136225c2a044538185d7866d307118 is 50, key is test_row_0/C:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:27,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742039_1215 (size=13085) 2024-11-09T21:16:27,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-09T21:16:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:27,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T21:16:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:27,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:27,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T21:16:27,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099f899ba604854f0fa933fa70bdbe903e_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186987023/Put/seqid=0 2024-11-09T21:16:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:27,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. as already flushing 2024-11-09T21:16:27,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742040_1216 (size=12454) 2024-11-09T21:16:27,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:27,686 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099f899ba604854f0fa933fa70bdbe903e_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099f899ba604854f0fa933fa70bdbe903e_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:27,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0dca94a77bd74b36bd5959f8b0ff6722, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:27,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0dca94a77bd74b36bd5959f8b0ff6722 is 175, key is test_row_0/A:col10/1731186987023/Put/seqid=0 2024-11-09T21:16:27,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742041_1217 (size=31255) 2024-11-09T21:16:27,736 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187047733, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,736 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187047734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187047736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,788 INFO [Thread-622 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-622. 2024-11-09T21:16:27,788 DEBUG [Thread-622 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:27,788 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,788 DEBUG [Thread-622 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,788 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:27,790 INFO [Thread-628 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-628. 2024-11-09T21:16:27,790 DEBUG [Thread-628 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:27,790 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,790 DEBUG [Thread-628 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,790 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:27,791 INFO [Thread-626 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-626. 2024-11-09T21:16:27,792 DEBUG [Thread-626 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:27,792 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,792 DEBUG [Thread-626 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,792 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:27,794 INFO [Thread-624 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-624. 2024-11-09T21:16:27,794 DEBUG [Thread-624 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:27,794 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,794 DEBUG [Thread-624 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:27,794 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:27,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187047846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187047847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187047846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,847 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:27,991 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#174 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:27,991 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/4751a31ebd244b3493bbca54b0b0bc6f is 175, key is test_row_0/A:col10/1731186986658/Put/seqid=0 2024-11-09T21:16:27,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742042_1218 (size=32039) 2024-11-09T21:16:28,026 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/9e136225c2a044538185d7866d307118 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/9e136225c2a044538185d7866d307118 2024-11-09T21:16:28,034 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 9e136225c2a044538185d7866d307118(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:28,034 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:28,034 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186987534; duration=0sec 2024-11-09T21:16:28,035 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:28,035 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:28,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,057 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187048057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187048057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187048057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,058 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,099 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=378, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0dca94a77bd74b36bd5959f8b0ff6722 2024-11-09T21:16:28,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/3414602bf8374d0fb21549f24d582613 is 50, key is test_row_0/B:col10/1731186987023/Put/seqid=0 2024-11-09T21:16:28,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742043_1219 (size=12301) 2024-11-09T21:16:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T21:16:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187048367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187048367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187048367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,369 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,410 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/4751a31ebd244b3493bbca54b0b0bc6f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4751a31ebd244b3493bbca54b0b0bc6f 2024-11-09T21:16:28,417 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into 4751a31ebd244b3493bbca54b0b0bc6f(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:28,417 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:28,417 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186987534; duration=0sec 2024-11-09T21:16:28,417 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:28,417 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:28,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/3414602bf8374d0fb21549f24d582613 2024-11-09T21:16:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/160a0e5f266147e6a96345141238e87e is 50, key is test_row_0/C:col10/1731186987023/Put/seqid=0 2024-11-09T21:16:28,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742044_1220 (size=12301) 2024-11-09T21:16:28,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35524 deadline: 1731187048686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,687 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35502 deadline: 1731187048697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,697 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in aeea20555c2d70c5adac9bfee3363c0e of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,698 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35520 deadline: 1731187048876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35514 deadline: 1731187048877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:35500 deadline: 1731187048877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:28,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:16:28,927 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/160a0e5f266147e6a96345141238e87e 2024-11-09T21:16:28,932 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/0dca94a77bd74b36bd5959f8b0ff6722 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722 2024-11-09T21:16:28,936 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722, entries=150, sequenceid=378, filesize=30.5 K 2024-11-09T21:16:28,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/3414602bf8374d0fb21549f24d582613 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3414602bf8374d0fb21549f24d582613 2024-11-09T21:16:28,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3414602bf8374d0fb21549f24d582613, entries=150, sequenceid=378, filesize=12.0 K 2024-11-09T21:16:28,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/160a0e5f266147e6a96345141238e87e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/160a0e5f266147e6a96345141238e87e 2024-11-09T21:16:28,945 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/160a0e5f266147e6a96345141238e87e, entries=150, sequenceid=378, filesize=12.0 K 2024-11-09T21:16:28,945 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for aeea20555c2d70c5adac9bfee3363c0e in 1326ms, sequenceid=378, compaction requested=false 2024-11-09T21:16:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:28,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-09T21:16:28,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-09T21:16:28,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-09T21:16:28,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9440 sec 2024-11-09T21:16:28,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 1.9510 sec 2024-11-09T21:16:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-09T21:16:29,157 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:29,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:29,888 INFO [Thread-611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-611. 2024-11-09T21:16:29,888 DEBUG [Thread-611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:29,888 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:29,888 DEBUG [Thread-611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:29,888 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:29,889 INFO [Thread-615 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-615. 2024-11-09T21:16:29,889 DEBUG [Thread-615 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:29,889 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:29,889 DEBUG [Thread-615 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:29,889 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:29,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:16:29,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:29,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:29,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:29,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:29,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:29,890 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:29,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fb1612930eb74162a75fe1e3bc74f181_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:29,901 INFO [Thread-617 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-617. 2024-11-09T21:16:29,901 DEBUG [Thread-617 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:29,901 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:29,901 DEBUG [Thread-617 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:29,901 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:29,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742045_1221 (size=12454) 2024-11-09T21:16:29,906 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:29,911 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109fb1612930eb74162a75fe1e3bc74f181_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb1612930eb74162a75fe1e3bc74f181_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:29,912 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f83942fda2614d0ab7fbaad104c14eea, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:29,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f83942fda2614d0ab7fbaad104c14eea is 175, key is test_row_0/A:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:29,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742046_1222 (size=31255) 2024-11-09T21:16:29,919 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=395, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f83942fda2614d0ab7fbaad104c14eea 2024-11-09T21:16:29,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/c358a1d5717c4a3fa64771b566490a75 is 50, key is test_row_0/B:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:29,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742047_1223 (size=12301) 2024-11-09T21:16:30,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/c358a1d5717c4a3fa64771b566490a75 2024-11-09T21:16:30,348 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/ee828226548d4dbeac2f89f56c69ce29 is 50, key is test_row_0/C:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:30,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742048_1224 (size=12301) 2024-11-09T21:16:30,754 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=395 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/ee828226548d4dbeac2f89f56c69ce29 2024-11-09T21:16:30,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/f83942fda2614d0ab7fbaad104c14eea as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea 2024-11-09T21:16:30,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea, entries=150, sequenceid=395, filesize=30.5 K 2024-11-09T21:16:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/c358a1d5717c4a3fa64771b566490a75 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c358a1d5717c4a3fa64771b566490a75 2024-11-09T21:16:30,772 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c358a1d5717c4a3fa64771b566490a75, entries=150, sequenceid=395, filesize=12.0 K 2024-11-09T21:16:30,773 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/ee828226548d4dbeac2f89f56c69ce29 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/ee828226548d4dbeac2f89f56c69ce29 2024-11-09T21:16:30,778 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/ee828226548d4dbeac2f89f56c69ce29, entries=150, sequenceid=395, filesize=12.0 K 2024-11-09T21:16:30,780 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=6.71 KB/6870 for aeea20555c2d70c5adac9bfee3363c0e in 890ms, sequenceid=395, compaction requested=true 2024-11-09T21:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:30,780 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:30,780 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:30,780 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store aeea20555c2d70c5adac9bfee3363c0e:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:30,781 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:30,781 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:30,781 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:30,781 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/A is initiating minor compaction (all files) 2024-11-09T21:16:30,781 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/B is initiating minor compaction (all files) 2024-11-09T21:16:30,782 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/A in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:30,782 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/B in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:30,782 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4751a31ebd244b3493bbca54b0b0bc6f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=92.3 K 2024-11-09T21:16:30,782 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/565d2bddaf3445cfaa62f108b3699733, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3414602bf8374d0fb21549f24d582613, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c358a1d5717c4a3fa64771b566490a75] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=36.8 K 2024-11-09T21:16:30,782 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:30,782 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4751a31ebd244b3493bbca54b0b0bc6f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea] 2024-11-09T21:16:30,782 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 565d2bddaf3445cfaa62f108b3699733, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731186986655 2024-11-09T21:16:30,782 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4751a31ebd244b3493bbca54b0b0bc6f, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731186986655 2024-11-09T21:16:30,783 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0dca94a77bd74b36bd5959f8b0ff6722, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731186987015 2024-11-09T21:16:30,783 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3414602bf8374d0fb21549f24d582613, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731186987015 2024-11-09T21:16:30,783 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f83942fda2614d0ab7fbaad104c14eea, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731186987722 2024-11-09T21:16:30,783 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c358a1d5717c4a3fa64771b566490a75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731186987722 2024-11-09T21:16:30,798 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#B#compaction#183 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:30,798 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/4c9b7990ca5444678777be94fe341e49 is 50, key is test_row_0/B:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:30,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742049_1225 (size=13187) 2024-11-09T21:16:30,813 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:30,829 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411096ff9f18cb08340e7bac5e86ee5aaf941_aeea20555c2d70c5adac9bfee3363c0e store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:30,891 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411096ff9f18cb08340e7bac5e86ee5aaf941_aeea20555c2d70c5adac9bfee3363c0e, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:30,891 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411096ff9f18cb08340e7bac5e86ee5aaf941_aeea20555c2d70c5adac9bfee3363c0e because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:30,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742050_1226 (size=4469) 2024-11-09T21:16:30,906 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#A#compaction#184 average throughput is 0.26 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:30,907 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/462975015e3d44769eff8444a584e4e2 is 175, key is test_row_0/A:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:30,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742051_1227 (size=32141) 2024-11-09T21:16:30,926 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/462975015e3d44769eff8444a584e4e2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/462975015e3d44769eff8444a584e4e2 2024-11-09T21:16:30,933 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/A of aeea20555c2d70c5adac9bfee3363c0e into 462975015e3d44769eff8444a584e4e2(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:30,933 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:30,933 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/A, priority=13, startTime=1731186990780; duration=0sec 2024-11-09T21:16:30,933 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:30,933 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:A 2024-11-09T21:16:30,933 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:30,934 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:30,934 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): aeea20555c2d70c5adac9bfee3363c0e/C is initiating minor compaction (all files) 2024-11-09T21:16:30,934 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of aeea20555c2d70c5adac9bfee3363c0e/C in TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:30,934 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/9e136225c2a044538185d7866d307118, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/160a0e5f266147e6a96345141238e87e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/ee828226548d4dbeac2f89f56c69ce29] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp, totalSize=36.8 K 2024-11-09T21:16:30,934 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e136225c2a044538185d7866d307118, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=358, earliestPutTs=1731186986655 2024-11-09T21:16:30,934 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 160a0e5f266147e6a96345141238e87e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731186987015 2024-11-09T21:16:30,935 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee828226548d4dbeac2f89f56c69ce29, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=395, earliestPutTs=1731186987722 2024-11-09T21:16:30,948 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): aeea20555c2d70c5adac9bfee3363c0e#C#compaction#185 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:30,949 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/8afc504111ac48d6b092aa7204541f87 is 50, key is test_row_0/C:col10/1731186987734/Put/seqid=0 2024-11-09T21:16:30,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742052_1228 (size=13187) 2024-11-09T21:16:30,976 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/8afc504111ac48d6b092aa7204541f87 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8afc504111ac48d6b092aa7204541f87 2024-11-09T21:16:30,985 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/C of aeea20555c2d70c5adac9bfee3363c0e into 8afc504111ac48d6b092aa7204541f87(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:30,985 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:30,985 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/C, priority=13, startTime=1731186990781; duration=0sec 2024-11-09T21:16:30,985 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:30,985 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:C 2024-11-09T21:16:31,215 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/4c9b7990ca5444678777be94fe341e49 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4c9b7990ca5444678777be94fe341e49 2024-11-09T21:16:31,222 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in aeea20555c2d70c5adac9bfee3363c0e/B of aeea20555c2d70c5adac9bfee3363c0e into 4c9b7990ca5444678777be94fe341e49(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:31,222 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for aeea20555c2d70c5adac9bfee3363c0e: 2024-11-09T21:16:31,222 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., storeName=aeea20555c2d70c5adac9bfee3363c0e/B, priority=13, startTime=1731186990780; duration=0sec 2024-11-09T21:16:31,222 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:31,222 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: aeea20555c2d70c5adac9bfee3363c0e:B 2024-11-09T21:16:32,700 INFO [Thread-619 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-619. 2024-11-09T21:16:32,700 DEBUG [Thread-619 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:32,700 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:32,700 DEBUG [Thread-619 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:32,700 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:32,730 INFO [Thread-613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-613. 2024-11-09T21:16:32,730 DEBUG [Thread-613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:32,730 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:32,730 DEBUG [Thread-613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:32,730 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 67 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2638 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2528 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1117 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3338 rows 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1117 2024-11-09T21:16:32,731 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3340 rows 2024-11-09T21:16:32,732 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:16:32,732 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:16:32,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:32,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:32,732 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:32,734 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T21:16:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:32,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T21:16:32,740 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186992740"}]},"ts":"1731186992740"} 2024-11-09T21:16:32,742 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T21:16:32,742 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T21:16:32,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:16:32,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, UNASSIGN}] 2024-11-09T21:16:32,746 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, UNASSIGN 2024-11-09T21:16:32,747 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:32,749 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, UNASSIGN because future has completed 2024-11-09T21:16:32,749 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:16:32,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:16:32,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T21:16:32,903 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:32,903 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:16:32,903 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing aeea20555c2d70c5adac9bfee3363c0e, disabling compactions & flushes 2024-11-09T21:16:32,903 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:32,903 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:32,903 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. after waiting 0 ms 2024-11-09T21:16:32,903 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:32,904 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing aeea20555c2d70c5adac9bfee3363c0e 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-09T21:16:32,904 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=A 2024-11-09T21:16:32,904 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:32,904 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=B 2024-11-09T21:16:32,904 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:32,904 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK aeea20555c2d70c5adac9bfee3363c0e, store=C 2024-11-09T21:16:32,904 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:32,916 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092da9cdc6aa214f96a3806e39b1827939_aeea20555c2d70c5adac9bfee3363c0e is 50, key is test_row_0/A:col10/1731186992727/Put/seqid=0 2024-11-09T21:16:32,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742053_1229 (size=9914) 2024-11-09T21:16:33,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T21:16:33,223 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:16:33,321 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:33,324 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411092da9cdc6aa214f96a3806e39b1827939_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092da9cdc6aa214f96a3806e39b1827939_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:33,325 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/87b2988dba144be4b0da9d2230851a6f, store: [table=TestAcidGuarantees family=A region=aeea20555c2d70c5adac9bfee3363c0e] 2024-11-09T21:16:33,325 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/87b2988dba144be4b0da9d2230851a6f is 175, key is test_row_0/A:col10/1731186992727/Put/seqid=0 2024-11-09T21:16:33,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742054_1230 (size=22561) 2024-11-09T21:16:33,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T21:16:33,730 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=6.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/87b2988dba144be4b0da9d2230851a6f 2024-11-09T21:16:33,739 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/b45f082d9b474654a783d5e753dc6198 is 50, key is test_row_0/B:col10/1731186992727/Put/seqid=0 2024-11-09T21:16:33,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742055_1231 (size=9857) 2024-11-09T21:16:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T21:16:34,144 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/b45f082d9b474654a783d5e753dc6198 2024-11-09T21:16:34,151 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/e61b86c98bcc4fc1a0f614a13adf7897 is 50, key is test_row_0/C:col10/1731186992727/Put/seqid=0 2024-11-09T21:16:34,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742056_1232 (size=9857) 2024-11-09T21:16:34,556 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/e61b86c98bcc4fc1a0f614a13adf7897 2024-11-09T21:16:34,564 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/A/87b2988dba144be4b0da9d2230851a6f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/87b2988dba144be4b0da9d2230851a6f 2024-11-09T21:16:34,570 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/87b2988dba144be4b0da9d2230851a6f, entries=100, sequenceid=404, filesize=22.0 K 2024-11-09T21:16:34,571 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/B/b45f082d9b474654a783d5e753dc6198 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b45f082d9b474654a783d5e753dc6198 2024-11-09T21:16:34,576 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b45f082d9b474654a783d5e753dc6198, entries=100, sequenceid=404, filesize=9.6 K 2024-11-09T21:16:34,576 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/.tmp/C/e61b86c98bcc4fc1a0f614a13adf7897 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/e61b86c98bcc4fc1a0f614a13adf7897 2024-11-09T21:16:34,580 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/e61b86c98bcc4fc1a0f614a13adf7897, entries=100, sequenceid=404, filesize=9.6 K 2024-11-09T21:16:34,581 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for aeea20555c2d70c5adac9bfee3363c0e in 1678ms, sequenceid=404, compaction requested=false 2024-11-09T21:16:34,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/fceba0c46f8a4218986a10acf5556aa7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/c10fec656d61469089071aa7af503786, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4a4659f879de4cc09f76e9e5c23b1319, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/239c3c963aee453e83c6788bdc6ef9a9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cb83607ee22446e79b2a862a8947f0c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8cde54a816b048b5aa226fce3c7318bd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4751a31ebd244b3493bbca54b0b0bc6f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea] to archive 2024-11-09T21:16:34,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:16:34,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/89d261a47b7d4f34bf5fd4bfea214804 2024-11-09T21:16:34,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/a6f2bb25d2244fa5914b7344caed3320 2024-11-09T21:16:34,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/fceba0c46f8a4218986a10acf5556aa7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/fceba0c46f8a4218986a10acf5556aa7 2024-11-09T21:16:34,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/79775f79641944fe9de2348814702201 2024-11-09T21:16:34,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/bfac464002ee431a8e7f4d73db6b6fbf 2024-11-09T21:16:34,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/b64c8f248e3a4ebcad7435b816cd413f 2024-11-09T21:16:34,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/c10fec656d61469089071aa7af503786 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/c10fec656d61469089071aa7af503786 2024-11-09T21:16:34,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/5ffa13e1ae1348c7b5daa0572fe36736 2024-11-09T21:16:34,592 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/07d1ec0bc24347cda4848d375b699ca4 2024-11-09T21:16:34,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4a4659f879de4cc09f76e9e5c23b1319 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4a4659f879de4cc09f76e9e5c23b1319 2024-11-09T21:16:34,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/eb3cd2c0b9f54f6b8c32164a2546cdf1 2024-11-09T21:16:34,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/76742d992ec94f52ae4dfca83d1b2e66 2024-11-09T21:16:34,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/239c3c963aee453e83c6788bdc6ef9a9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/239c3c963aee453e83c6788bdc6ef9a9 2024-11-09T21:16:34,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/183dd99d0bcc48428aa5bd5797613eec 2024-11-09T21:16:34,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0b1b9f552157490487141d71af0e2198 2024-11-09T21:16:34,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8dd25477a79642d9be3f2144216c8e43 2024-11-09T21:16:34,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/96b8c9a3dac343859224012bf52fff4a 2024-11-09T21:16:34,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cb83607ee22446e79b2a862a8947f0c0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cb83607ee22446e79b2a862a8947f0c0 2024-11-09T21:16:34,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/3dc2f487d8204a59a744c9bf57118129 2024-11-09T21:16:34,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8cde54a816b048b5aa226fce3c7318bd to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/8cde54a816b048b5aa226fce3c7318bd 2024-11-09T21:16:34,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f31a7ef86bb4450296d1d47c62f5bfcc 2024-11-09T21:16:34,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/cfde799599c84d11a7281fbb2843db0a 2024-11-09T21:16:34,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0fa998f9814e41aaa0751ec26801bc1d 2024-11-09T21:16:34,606 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4751a31ebd244b3493bbca54b0b0bc6f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/4751a31ebd244b3493bbca54b0b0bc6f 2024-11-09T21:16:34,607 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/0dca94a77bd74b36bd5959f8b0ff6722 2024-11-09T21:16:34,608 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/f83942fda2614d0ab7fbaad104c14eea 2024-11-09T21:16:34,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/079ab2e54d764781bd19387d68b02d57, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/83ef211f26874855bd462a764c6928fd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/1afa62833cfe4817a300748ce89bcc39, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5fd33241b74941edb85ade8ebc0c9cab, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/235409bd313b44c9a764308e19e00665, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/34c318a94e204a56814ea746c34c3f44, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c5548bb0a3fd4495b15dd1c9af515f67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/9eaebb070cae47ca9d1a44f8754c3cb2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/cc23bc235fcf455eae83ab531000dc36, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/98139b1ffea6494d8a6a9d802dc7f18b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4db50baaaf2e43c789632efc0da45821, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/86fcffa2fdda46b4a2c1c81f099cfbd4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/de105a102cda4f41a65c46657fdba529, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/f0c2e4b7408e4d99859118e0f0f9e6cf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5cd7baffb1114fe8bf2f3e1c5777b2d4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b211e934c6a44dbf93ed0c20734a2c9b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/dea9b50947b24d1fa7e1205b69bb83b0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/90723ee9f98d4851a4b21adb46f54acb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/188dd6e5ddc340188af987fc7527cd47, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/515febcbda5c499796310441b9362196, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/fa71abc496ff40ad900fb291b5e60c04, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/14e915a706fd454aa52a936994f43ad3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/565d2bddaf3445cfaa62f108b3699733, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3982256cbc254dea8fa2ed3b11f28db5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3414602bf8374d0fb21549f24d582613, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c358a1d5717c4a3fa64771b566490a75] to archive 2024-11-09T21:16:34,610 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:16:34,611 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/079ab2e54d764781bd19387d68b02d57 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/079ab2e54d764781bd19387d68b02d57 2024-11-09T21:16:34,612 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/83ef211f26874855bd462a764c6928fd to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/83ef211f26874855bd462a764c6928fd 2024-11-09T21:16:34,613 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/1afa62833cfe4817a300748ce89bcc39 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/1afa62833cfe4817a300748ce89bcc39 2024-11-09T21:16:34,614 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5fd33241b74941edb85ade8ebc0c9cab to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5fd33241b74941edb85ade8ebc0c9cab 2024-11-09T21:16:34,615 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/235409bd313b44c9a764308e19e00665 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/235409bd313b44c9a764308e19e00665 2024-11-09T21:16:34,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/34c318a94e204a56814ea746c34c3f44 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/34c318a94e204a56814ea746c34c3f44 2024-11-09T21:16:34,617 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c5548bb0a3fd4495b15dd1c9af515f67 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c5548bb0a3fd4495b15dd1c9af515f67 2024-11-09T21:16:34,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/9eaebb070cae47ca9d1a44f8754c3cb2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/9eaebb070cae47ca9d1a44f8754c3cb2 2024-11-09T21:16:34,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/cc23bc235fcf455eae83ab531000dc36 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/cc23bc235fcf455eae83ab531000dc36 2024-11-09T21:16:34,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/98139b1ffea6494d8a6a9d802dc7f18b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/98139b1ffea6494d8a6a9d802dc7f18b 2024-11-09T21:16:34,620 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4db50baaaf2e43c789632efc0da45821 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4db50baaaf2e43c789632efc0da45821 2024-11-09T21:16:34,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/86fcffa2fdda46b4a2c1c81f099cfbd4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/86fcffa2fdda46b4a2c1c81f099cfbd4 2024-11-09T21:16:34,622 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/de105a102cda4f41a65c46657fdba529 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/de105a102cda4f41a65c46657fdba529 2024-11-09T21:16:34,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/f0c2e4b7408e4d99859118e0f0f9e6cf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/f0c2e4b7408e4d99859118e0f0f9e6cf 2024-11-09T21:16:34,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5cd7baffb1114fe8bf2f3e1c5777b2d4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/5cd7baffb1114fe8bf2f3e1c5777b2d4 2024-11-09T21:16:34,625 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b211e934c6a44dbf93ed0c20734a2c9b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b211e934c6a44dbf93ed0c20734a2c9b 2024-11-09T21:16:34,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/dea9b50947b24d1fa7e1205b69bb83b0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/dea9b50947b24d1fa7e1205b69bb83b0 2024-11-09T21:16:34,626 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/90723ee9f98d4851a4b21adb46f54acb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/90723ee9f98d4851a4b21adb46f54acb 2024-11-09T21:16:34,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/188dd6e5ddc340188af987fc7527cd47 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/188dd6e5ddc340188af987fc7527cd47 2024-11-09T21:16:34,629 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/515febcbda5c499796310441b9362196 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/515febcbda5c499796310441b9362196 2024-11-09T21:16:34,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/fa71abc496ff40ad900fb291b5e60c04 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/fa71abc496ff40ad900fb291b5e60c04 2024-11-09T21:16:34,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/14e915a706fd454aa52a936994f43ad3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/14e915a706fd454aa52a936994f43ad3 2024-11-09T21:16:34,632 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/565d2bddaf3445cfaa62f108b3699733 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/565d2bddaf3445cfaa62f108b3699733 2024-11-09T21:16:34,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3982256cbc254dea8fa2ed3b11f28db5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3982256cbc254dea8fa2ed3b11f28db5 2024-11-09T21:16:34,634 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3414602bf8374d0fb21549f24d582613 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/3414602bf8374d0fb21549f24d582613 2024-11-09T21:16:34,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c358a1d5717c4a3fa64771b566490a75 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/c358a1d5717c4a3fa64771b566490a75 2024-11-09T21:16:34,636 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5e085565f6ae4ab5ace384f9bf884a67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09dbe0b53dda40b0b9c10dba98a6088d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8118fb2d8345474784133c664284c45f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/6d12356938864b07859f72916d6bbe63, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/bae48035fef94befac04361914e6b631, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/388baee743674de99228b056d79b7965, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/4c8e02ad587a40c5ad93d7e16fbfcf8f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c7b1790e611349f187d907b686ca5753, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/b67770bd1fc544df99a57ae70a2ca1d4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c1373e10b11949938ce61caa153f54ce, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/67aa10afc6434afb822b1877dea13a23, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/13210335ef7d43e9a9ca1fe1930f2f4d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5eda425e01bd435abc68c01d5cefd311, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/2f4295b507ca4d7f9e3bea3a6564a685, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09442dce1d3b41888ea5d039ada18829, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/f3303cd4e9da4191a7be3263754e639e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/0971bad10544453888417af3d652074c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/7fa46f38383443808dd0f7c736abe351, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5bcecb79695249b5bbaf52f8be28b1ed, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/91f9db7c0b1e47fb8eddec79a360a8cb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/832bcec4dac849dd9d403ea15ee60275, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/fcccbceec66e4c76846da65c0b23a410, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/9e136225c2a044538185d7866d307118, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/57c4b937574143db9f1da681dc6e0653, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/160a0e5f266147e6a96345141238e87e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/ee828226548d4dbeac2f89f56c69ce29] to archive 2024-11-09T21:16:34,637 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:16:34,638 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5e085565f6ae4ab5ace384f9bf884a67 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5e085565f6ae4ab5ace384f9bf884a67 2024-11-09T21:16:34,639 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09dbe0b53dda40b0b9c10dba98a6088d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09dbe0b53dda40b0b9c10dba98a6088d 2024-11-09T21:16:34,641 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8118fb2d8345474784133c664284c45f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8118fb2d8345474784133c664284c45f 2024-11-09T21:16:34,642 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/6d12356938864b07859f72916d6bbe63 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/6d12356938864b07859f72916d6bbe63 2024-11-09T21:16:34,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/bae48035fef94befac04361914e6b631 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/bae48035fef94befac04361914e6b631 2024-11-09T21:16:34,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/388baee743674de99228b056d79b7965 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/388baee743674de99228b056d79b7965 2024-11-09T21:16:34,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/4c8e02ad587a40c5ad93d7e16fbfcf8f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/4c8e02ad587a40c5ad93d7e16fbfcf8f 2024-11-09T21:16:34,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c7b1790e611349f187d907b686ca5753 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c7b1790e611349f187d907b686ca5753 2024-11-09T21:16:34,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/b67770bd1fc544df99a57ae70a2ca1d4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/b67770bd1fc544df99a57ae70a2ca1d4 2024-11-09T21:16:34,647 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c1373e10b11949938ce61caa153f54ce to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/c1373e10b11949938ce61caa153f54ce 2024-11-09T21:16:34,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/67aa10afc6434afb822b1877dea13a23 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/67aa10afc6434afb822b1877dea13a23 2024-11-09T21:16:34,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/13210335ef7d43e9a9ca1fe1930f2f4d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/13210335ef7d43e9a9ca1fe1930f2f4d 2024-11-09T21:16:34,649 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5eda425e01bd435abc68c01d5cefd311 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5eda425e01bd435abc68c01d5cefd311 2024-11-09T21:16:34,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/2f4295b507ca4d7f9e3bea3a6564a685 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/2f4295b507ca4d7f9e3bea3a6564a685 2024-11-09T21:16:34,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09442dce1d3b41888ea5d039ada18829 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/09442dce1d3b41888ea5d039ada18829 2024-11-09T21:16:34,652 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/f3303cd4e9da4191a7be3263754e639e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/f3303cd4e9da4191a7be3263754e639e 2024-11-09T21:16:34,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/0971bad10544453888417af3d652074c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/0971bad10544453888417af3d652074c 2024-11-09T21:16:34,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/7fa46f38383443808dd0f7c736abe351 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/7fa46f38383443808dd0f7c736abe351 2024-11-09T21:16:34,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5bcecb79695249b5bbaf52f8be28b1ed to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/5bcecb79695249b5bbaf52f8be28b1ed 2024-11-09T21:16:34,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/91f9db7c0b1e47fb8eddec79a360a8cb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/91f9db7c0b1e47fb8eddec79a360a8cb 2024-11-09T21:16:34,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/832bcec4dac849dd9d403ea15ee60275 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/832bcec4dac849dd9d403ea15ee60275 2024-11-09T21:16:34,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/fcccbceec66e4c76846da65c0b23a410 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/fcccbceec66e4c76846da65c0b23a410 2024-11-09T21:16:34,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/9e136225c2a044538185d7866d307118 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/9e136225c2a044538185d7866d307118 2024-11-09T21:16:34,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/57c4b937574143db9f1da681dc6e0653 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/57c4b937574143db9f1da681dc6e0653 2024-11-09T21:16:34,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/160a0e5f266147e6a96345141238e87e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/160a0e5f266147e6a96345141238e87e 2024-11-09T21:16:34,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/ee828226548d4dbeac2f89f56c69ce29 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/ee828226548d4dbeac2f89f56c69ce29 2024-11-09T21:16:34,665 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/recovered.edits/407.seqid, newMaxSeqId=407, maxSeqId=4 2024-11-09T21:16:34,666 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. 2024-11-09T21:16:34,666 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for aeea20555c2d70c5adac9bfee3363c0e: Waiting for close lock at 1731186992903Running coprocessor pre-close hooks at 1731186992903Disabling compacts and flushes for region at 1731186992903Disabling writes for close at 1731186992903Obtaining lock to block concurrent updates at 1731186992904 (+1 ms)Preparing flush snapshotting stores in aeea20555c2d70c5adac9bfee3363c0e at 1731186992904Finished memstore snapshotting TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731186992905 (+1 ms)Flushing stores of TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e. at 1731186992906 (+1 ms)Flushing aeea20555c2d70c5adac9bfee3363c0e/A: creating writer at 1731186992906Flushing mob file aeea20555c2d70c5adac9bfee3363c0e/A: appending metadata at 1731186992916 (+10 ms)Flushing mob file aeea20555c2d70c5adac9bfee3363c0e/A: closing flushed file at 1731186992916Flushing aeea20555c2d70c5adac9bfee3363c0e/A: appending metadata at 1731186993325 (+409 ms)Flushing aeea20555c2d70c5adac9bfee3363c0e/A: closing flushed file at 1731186993325Flushing aeea20555c2d70c5adac9bfee3363c0e/B: creating writer at 1731186993736 (+411 ms)Flushing aeea20555c2d70c5adac9bfee3363c0e/B: appending metadata at 1731186993739 (+3 ms)Flushing aeea20555c2d70c5adac9bfee3363c0e/B: closing flushed file at 1731186993739Flushing aeea20555c2d70c5adac9bfee3363c0e/C: creating writer at 1731186994148 (+409 ms)Flushing aeea20555c2d70c5adac9bfee3363c0e/C: appending metadata at 1731186994150 (+2 ms)Flushing aeea20555c2d70c5adac9bfee3363c0e/C: closing flushed file at 1731186994150Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d14cc66: reopening flushed file at 1731186994563 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@877b1e9: reopening flushed file at 1731186994570 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@424c1ab5: reopening flushed file at 1731186994576 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for aeea20555c2d70c5adac9bfee3363c0e in 1678ms, sequenceid=404, compaction requested=false at 1731186994581 (+5 ms)Writing region close event to WAL at 1731186994661 (+80 ms)Running coprocessor post-close hooks at 1731186994666 (+5 ms)Closed at 1731186994666 2024-11-09T21:16:34,668 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,669 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=aeea20555c2d70c5adac9bfee3363c0e, regionState=CLOSED 2024-11-09T21:16:34,670 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:16:34,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T21:16:34,673 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure aeea20555c2d70c5adac9bfee3363c0e, server=9514d6d5e5be,38413,1731186934857 in 1.9220 sec 2024-11-09T21:16:34,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-09T21:16:34,675 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=aeea20555c2d70c5adac9bfee3363c0e, UNASSIGN in 1.9280 sec 2024-11-09T21:16:34,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T21:16:34,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9330 sec 2024-11-09T21:16:34,679 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186994679"}]},"ts":"1731186994679"} 2024-11-09T21:16:34,681 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T21:16:34,681 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T21:16:34,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9480 sec 2024-11-09T21:16:34,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-09T21:16:34,887 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:34,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T21:16:34,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:34,891 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:34,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T21:16:34,892 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:34,898 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,902 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/recovered.edits] 2024-11-09T21:16:34,905 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/462975015e3d44769eff8444a584e4e2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/462975015e3d44769eff8444a584e4e2 2024-11-09T21:16:34,906 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/87b2988dba144be4b0da9d2230851a6f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/A/87b2988dba144be4b0da9d2230851a6f 2024-11-09T21:16:34,908 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4c9b7990ca5444678777be94fe341e49 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/4c9b7990ca5444678777be94fe341e49 2024-11-09T21:16:34,909 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b45f082d9b474654a783d5e753dc6198 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/B/b45f082d9b474654a783d5e753dc6198 2024-11-09T21:16:34,911 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8afc504111ac48d6b092aa7204541f87 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/8afc504111ac48d6b092aa7204541f87 2024-11-09T21:16:34,912 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/e61b86c98bcc4fc1a0f614a13adf7897 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/C/e61b86c98bcc4fc1a0f614a13adf7897 2024-11-09T21:16:34,914 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/recovered.edits/407.seqid to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e/recovered.edits/407.seqid 2024-11-09T21:16:34,915 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,915 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T21:16:34,915 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T21:16:34,916 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T21:16:34,920 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110910013437f50f4f7e8162147f394b4ba7_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110910013437f50f4f7e8162147f394b4ba7_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,921 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110925555c3a496a475bb7867278e6fd5b79_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110925555c3a496a475bb7867278e6fd5b79_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,922 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092da9cdc6aa214f96a3806e39b1827939_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411092da9cdc6aa214f96a3806e39b1827939_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,923 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095620aa87d8e34867a8b9d1dd664a9347_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095620aa87d8e34867a8b9d1dd664a9347_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,924 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109596902c965454ec2b2f3018382a7eca4_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109596902c965454ec2b2f3018382a7eca4_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,925 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095a567d01f688428d8e1eb7ab189f425b_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095a567d01f688428d8e1eb7ab189f425b_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,926 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109637ec770b27d41e58809fc057d2f49b1_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109637ec770b27d41e58809fc057d2f49b1_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,927 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109641d94c62e9348ee97e1293ee7b18bc0_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109641d94c62e9348ee97e1293ee7b18bc0_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,928 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110965730fa65cc940a182fbfce16c65eb6f_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110965730fa65cc940a182fbfce16c65eb6f_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,929 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097c41567aaa9b4212aeb7ee4a7233a4b5_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411097c41567aaa9b4212aeb7ee4a7233a4b5_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,930 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099260c6ca9ab349d480b057ec364f2970_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099260c6ca9ab349d480b057ec364f2970_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,931 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099f899ba604854f0fa933fa70bdbe903e_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099f899ba604854f0fa933fa70bdbe903e_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,931 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ab9b1c46ec874f75b124776c3f21addd_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ab9b1c46ec874f75b124776c3f21addd_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,932 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109be6f7da92f9b4e16b4813cdbbd5c1175_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109be6f7da92f9b4e16b4813cdbbd5c1175_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,933 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109becfc144d0ee4c22b1b73230f3de4cce_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109becfc144d0ee4c22b1b73230f3de4cce_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,934 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c49bec34a23e4e9894e318c93ba41d0f_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c49bec34a23e4e9894e318c93ba41d0f_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,935 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d09fd5cfbd7e463bab3f3df498968e9a_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109d09fd5cfbd7e463bab3f3df498968e9a_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,936 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e89ffd914fe54fad84dda78e31a2535a_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e89ffd914fe54fad84dda78e31a2535a_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,937 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb1612930eb74162a75fe1e3bc74f181_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fb1612930eb74162a75fe1e3bc74f181_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,938 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fc311a39268b4bec8bc70c7cb3fa6097_aeea20555c2d70c5adac9bfee3363c0e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109fc311a39268b4bec8bc70c7cb3fa6097_aeea20555c2d70c5adac9bfee3363c0e 2024-11-09T21:16:34,938 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T21:16:34,940 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:34,943 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T21:16:34,945 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T21:16:34,946 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:34,946 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T21:16:34,946 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731186994946"}]},"ts":"9223372036854775807"} 2024-11-09T21:16:34,948 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T21:16:34,948 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => aeea20555c2d70c5adac9bfee3363c0e, NAME => 'TestAcidGuarantees,,1731186964585.aeea20555c2d70c5adac9bfee3363c0e.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T21:16:34,949 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T21:16:34,949 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731186994949"}]},"ts":"9223372036854775807"} 2024-11-09T21:16:34,951 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T21:16:34,952 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:34,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 65 msec 2024-11-09T21:16:34,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-09T21:16:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T21:16:34,997 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:35,008 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=252 (was 249) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1645226176_22 at /127.0.0.1:46340 [Waiting for operation #1095] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-71581069_22 at /127.0.0.1:48126 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-71581069_22 at /127.0.0.1:39316 [Waiting for operation #894] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1645226176_22 at /127.0.0.1:39308 [Waiting for operation #896] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1645226176_22 at /127.0.0.1:54380 [Waiting for operation #1010] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=402 (was 363) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3074 (was 3791) 2024-11-09T21:16:35,020 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=252, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=402, ProcessCount=11, AvailableMemoryMB=3074 2024-11-09T21:16:35,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:16:35,022 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:16:35,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T21:16:35,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:16:35,024 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:35,024 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-09T21:16:35,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-09T21:16:35,026 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:16:35,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742057_1233 (size=1061) 2024-11-09T21:16:35,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-09T21:16:35,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-09T21:16:35,439 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:16:35,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742058_1234 (size=53) 2024-11-09T21:16:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-09T21:16:35,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:16:35,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing c7ac4dbed742a563978c6ff5cf7dc21b, disabling compactions & flushes 2024-11-09T21:16:35,851 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:35,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:35,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. after waiting 0 ms 2024-11-09T21:16:35,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:35,851 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:35,851 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for c7ac4dbed742a563978c6ff5cf7dc21b: Waiting for close lock at 1731186995851Disabling compacts and flushes for region at 1731186995851Disabling writes for close at 1731186995851Writing region close event to WAL at 1731186995851Closed at 1731186995851 2024-11-09T21:16:35,854 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:16:35,855 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731186995854"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731186995854"}]},"ts":"1731186995854"} 2024-11-09T21:16:35,859 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:16:35,860 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:16:35,861 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186995860"}]},"ts":"1731186995860"} 2024-11-09T21:16:35,862 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T21:16:35,863 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, ASSIGN}] 2024-11-09T21:16:35,864 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, ASSIGN 2024-11-09T21:16:35,865 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, ASSIGN; state=OFFLINE, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=false 2024-11-09T21:16:36,015 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c7ac4dbed742a563978c6ff5cf7dc21b, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:36,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, ASSIGN because future has completed 2024-11-09T21:16:36,017 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:16:36,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-09T21:16:36,173 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:36,173 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:16:36,173 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,173 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:16:36,174 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,174 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,176 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,179 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:36,179 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7ac4dbed742a563978c6ff5cf7dc21b columnFamilyName A 2024-11-09T21:16:36,179 DEBUG [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:36,180 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.HStore(327): Store=c7ac4dbed742a563978c6ff5cf7dc21b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:36,180 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,181 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:36,182 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7ac4dbed742a563978c6ff5cf7dc21b columnFamilyName B 2024-11-09T21:16:36,182 DEBUG [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:36,182 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.HStore(327): Store=c7ac4dbed742a563978c6ff5cf7dc21b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:36,182 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,183 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:16:36,184 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7ac4dbed742a563978c6ff5cf7dc21b columnFamilyName C 2024-11-09T21:16:36,184 DEBUG [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:16:36,184 INFO [StoreOpener-c7ac4dbed742a563978c6ff5cf7dc21b-1 {}] regionserver.HStore(327): Store=c7ac4dbed742a563978c6ff5cf7dc21b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:16:36,184 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:36,184 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,185 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,186 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,187 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,187 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,187 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:16:36,188 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,190 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:16:36,191 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened c7ac4dbed742a563978c6ff5cf7dc21b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59327503, jitterRate=-0.11595131456851959}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:16:36,191 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:36,191 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for c7ac4dbed742a563978c6ff5cf7dc21b: Running coprocessor pre-open hook at 1731186996174Writing region info on filesystem at 1731186996174Initializing all the Stores at 1731186996175 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186996175Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186996176 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731186996176Cleaning up temporary data from old regions at 1731186996187 (+11 ms)Running coprocessor post-open hooks at 1731186996191 (+4 ms)Region opened successfully at 1731186996191 2024-11-09T21:16:36,192 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., pid=65, masterSystemTime=1731186996168 2024-11-09T21:16:36,194 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:36,194 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:36,194 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=c7ac4dbed742a563978c6ff5cf7dc21b, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:36,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:16:36,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-09T21:16:36,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 in 179 msec 2024-11-09T21:16:36,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-09T21:16:36,199 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, ASSIGN in 334 msec 2024-11-09T21:16:36,200 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:16:36,200 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731186996200"}]},"ts":"1731186996200"} 2024-11-09T21:16:36,201 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T21:16:36,202 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:16:36,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-09T21:16:37,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-09T21:16:37,177 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:37,178 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79eacb52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,178 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,179 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,180 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5168b413, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,180 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,181 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38852, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba61438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55028, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,185 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:16:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:16:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@448dbb8d 2024-11-09T21:16:37,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:16:37,187 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:16:37,188 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a054597, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,188 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,188 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,189 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@742bf6f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,190 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,190 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38872, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,191 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f38f50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,191 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79d1dc57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,191 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,192 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,192 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9ac6dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,194 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38876, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25b10e78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,196 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a525a86, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,196 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,196 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,197 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c57045, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,197 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,198 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,199 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38890, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ed5113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,200 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73c2c082, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,200 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,200 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,201 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23cd599d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,201 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,202 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,202 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38914, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,203 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c0da5b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29df300d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,204 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,205 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231a69db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,206 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,207 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38932, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,207 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b1e675, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,208 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54c38859, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,208 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,208 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,209 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,209 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e963aa7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,210 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,211 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38942, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52058890, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,212 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f3197c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,212 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,212 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,213 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7adb9a6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,213 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,214 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38950, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,215 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24b45bae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,216 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533cbeb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,216 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,216 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,217 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cf9f06a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,218 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,218 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38970, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2125ab2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,219 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23a14518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,219 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,219 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,220 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@670d18b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,221 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,222 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38988, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,222 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1aa4e91e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,223 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef84e3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,223 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:16:37,223 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:16:37,223 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:16:37,223 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa6330, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:16:37,224 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:37,225 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39006, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:16:37,225 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47f8a028, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:16:37,226 DEBUG [Thread-1099 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,226 DEBUG [Thread-1095 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,226 DEBUG [Thread-1112 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,226 DEBUG [Thread-1103 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,226 DEBUG [Thread-1106 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,227 DEBUG [Thread-1114 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55040, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,228 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,228 DEBUG [Thread-1114 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55056, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,229 DEBUG [Thread-1114 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:37,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,229 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55084, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55052, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55080, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,230 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,230 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:37,231 DEBUG [Thread-1108 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,231 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:37,231 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:37,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,232 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,233 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,233 DEBUG [Thread-1097 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,235 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,235 DEBUG [Thread-1101 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,236 DEBUG [Thread-1110 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:16:37,236 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:16:37,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:16:37,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55116, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,238 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:16:37,238 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:16:37,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:37,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:16:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:37,246 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:37,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/6d9194f102ca4556a3285721eb49a82a is 50, key is test_row_0/A:col10/1731186997240/Put/seqid=0 2024-11-09T21:16:37,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742059_1235 (size=12001) 2024-11-09T21:16:37,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187057279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/6d9194f102ca4556a3285721eb49a82a 2024-11-09T21:16:37,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186997246, max timestamp=1731186997284Num uniques 859; 2024-11-09T21:16:37,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186997246, max timestamp=1731186997284Num uniques 859; 2024-11-09T21:16:37,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:37,289 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:37,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186997246, max timestamp=1731186997284Num uniques 859; 2024-11-09T21:16:37,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:37,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d82d320109844ef58399413f984af2ef is 50, key is test_row_0/B:col10/1731186997240/Put/seqid=0 2024-11-09T21:16:37,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187057325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187057324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:37,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187057331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,339 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187057334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742060_1236 (size=12001) 2024-11-09T21:16:37,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187057397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187057436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,440 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187057437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187057447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187057447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:37,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:37,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:37,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187057606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187057647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187057647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187057656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187057656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:37,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d82d320109844ef58399413f984af2ef 2024-11-09T21:16:37,803 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/05b54913b1564db89bbce3ae77105dd0 is 50, key is test_row_0/C:col10/1731186997240/Put/seqid=0 2024-11-09T21:16:37,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742061_1237 (size=12001) 2024-11-09T21:16:37,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/05b54913b1564db89bbce3ae77105dd0 2024-11-09T21:16:37,816 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/6d9194f102ca4556a3285721eb49a82a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/6d9194f102ca4556a3285721eb49a82a 2024-11-09T21:16:37,830 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/6d9194f102ca4556a3285721eb49a82a, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T21:16:37,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d82d320109844ef58399413f984af2ef as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d82d320109844ef58399413f984af2ef 2024-11-09T21:16:37,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:37,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,847 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d82d320109844ef58399413f984af2ef, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T21:16:37,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/05b54913b1564db89bbce3ae77105dd0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/05b54913b1564db89bbce3ae77105dd0 2024-11-09T21:16:37,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/05b54913b1564db89bbce3ae77105dd0, entries=150, sequenceid=13, filesize=11.7 K 2024-11-09T21:16:37,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:37,857 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for c7ac4dbed742a563978c6ff5cf7dc21b in 615ms, sequenceid=13, compaction requested=false 2024-11-09T21:16:37,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:37,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:37,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T21:16:37,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:37,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:37,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:37,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:37,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:37,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:37,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3778005ebd5843ed95fc4032614c3deb is 50, key is test_row_0/A:col10/1731186997330/Put/seqid=0 2024-11-09T21:16:37,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742062_1238 (size=14341) 2024-11-09T21:16:37,936 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3778005ebd5843ed95fc4032614c3deb 2024-11-09T21:16:37,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0c1a87ad8ca643038e9c62d499aac003 is 50, key is test_row_0/B:col10/1731186997330/Put/seqid=0 2024-11-09T21:16:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742063_1239 (size=12001) 2024-11-09T21:16:37,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0c1a87ad8ca643038e9c62d499aac003 2024-11-09T21:16:37,963 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187057959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187057962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,967 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187057963, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187057966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187057966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:37,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:37,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/46bb94ec4f644fc98ba70fb07ab08c7e is 50, key is test_row_0/C:col10/1731186997330/Put/seqid=0 2024-11-09T21:16:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742064_1240 (size=12001) 2024-11-09T21:16:37,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/46bb94ec4f644fc98ba70fb07ab08c7e 2024-11-09T21:16:37,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3778005ebd5843ed95fc4032614c3deb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3778005ebd5843ed95fc4032614c3deb 2024-11-09T21:16:37,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:37,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:37,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3778005ebd5843ed95fc4032614c3deb, entries=200, sequenceid=50, filesize=14.0 K 2024-11-09T21:16:38,008 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0c1a87ad8ca643038e9c62d499aac003 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0c1a87ad8ca643038e9c62d499aac003 2024-11-09T21:16:38,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0c1a87ad8ca643038e9c62d499aac003, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T21:16:38,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/46bb94ec4f644fc98ba70fb07ab08c7e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/46bb94ec4f644fc98ba70fb07ab08c7e 2024-11-09T21:16:38,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/46bb94ec4f644fc98ba70fb07ab08c7e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-09T21:16:38,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c7ac4dbed742a563978c6ff5cf7dc21b in 114ms, sequenceid=50, compaction requested=false 2024-11-09T21:16:38,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:38,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:38,068 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:16:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:38,069 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:38,073 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/2bce6518876c43b1a3cae4aa1fd76741 is 50, key is test_row_0/A:col10/1731186997961/Put/seqid=0 2024-11-09T21:16:38,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742065_1241 (size=12001) 2024-11-09T21:16:38,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186998081, max timestamp=1731186998111Num uniques 859; 2024-11-09T21:16:38,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:38,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186998081, max timestamp=1731186998111Num uniques 859; 2024-11-09T21:16:38,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186998081, max timestamp=1731186998111Num uniques 859; 2024-11-09T21:16:38,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:38,114 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:38,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187058140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,147 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187058142, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,149 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:38,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187058146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,152 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187058256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187058256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187058256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,259 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:38,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:38,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187058466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187058466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187058466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187058477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187058477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/2bce6518876c43b1a3cae4aa1fd76741 2024-11-09T21:16:38,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/1fb1fdd1a11e4f6593d4230b60e3a449 is 50, key is test_row_0/B:col10/1731186997961/Put/seqid=0 2024-11-09T21:16:38,508 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742066_1242 (size=12001) 2024-11-09T21:16:38,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,764 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187058776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187058777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:38,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187058778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,783 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:38,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:38,910 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/1fb1fdd1a11e4f6593d4230b60e3a449 2024-11-09T21:16:38,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:38,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a0048701cf9b459fbe5f72253869a0b8 is 50, key is test_row_0/C:col10/1731186997961/Put/seqid=0 2024-11-09T21:16:38,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:38,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742067_1243 (size=12001) 2024-11-09T21:16:38,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a0048701cf9b459fbe5f72253869a0b8 2024-11-09T21:16:38,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/2bce6518876c43b1a3cae4aa1fd76741 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/2bce6518876c43b1a3cae4aa1fd76741 2024-11-09T21:16:38,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/2bce6518876c43b1a3cae4aa1fd76741, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:16:38,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/1fb1fdd1a11e4f6593d4230b60e3a449 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1fb1fdd1a11e4f6593d4230b60e3a449 2024-11-09T21:16:38,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1fb1fdd1a11e4f6593d4230b60e3a449, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:16:38,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a0048701cf9b459fbe5f72253869a0b8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a0048701cf9b459fbe5f72253869a0b8 2024-11-09T21:16:38,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a0048701cf9b459fbe5f72253869a0b8, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:16:38,945 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for c7ac4dbed742a563978c6ff5cf7dc21b in 877ms, sequenceid=61, compaction requested=true 2024-11-09T21:16:38,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:38,946 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:38,946 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:38,946 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:38,948 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:38,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:38,948 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:38,948 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,948 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/6d9194f102ca4556a3285721eb49a82a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3778005ebd5843ed95fc4032614c3deb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/2bce6518876c43b1a3cae4aa1fd76741] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.4 K 2024-11-09T21:16:38,948 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:38,948 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:38,948 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:38,948 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d82d320109844ef58399413f984af2ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0c1a87ad8ca643038e9c62d499aac003, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1fb1fdd1a11e4f6593d4230b60e3a449] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.2 K 2024-11-09T21:16:38,949 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d9194f102ca4556a3285721eb49a82a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731186997235 2024-11-09T21:16:38,949 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting d82d320109844ef58399413f984af2ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731186997235 2024-11-09T21:16:38,949 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c1a87ad8ca643038e9c62d499aac003, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731186997329 2024-11-09T21:16:38,949 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3778005ebd5843ed95fc4032614c3deb, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731186997323 2024-11-09T21:16:38,949 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fb1fdd1a11e4f6593d4230b60e3a449, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186997960 2024-11-09T21:16:38,949 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bce6518876c43b1a3cae4aa1fd76741, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186997960 2024-11-09T21:16:38,958 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#198 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:38,958 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:38,958 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b3e433ca19674f9e959c2c830df93a6e is 50, key is test_row_0/B:col10/1731186997961/Put/seqid=0 2024-11-09T21:16:38,958 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/5af3a23aac9445f4980b54c28c75908a is 50, key is test_row_0/A:col10/1731186997961/Put/seqid=0 2024-11-09T21:16:38,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742068_1244 (size=12104) 2024-11-09T21:16:38,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742069_1245 (size=12104) 2024-11-09T21:16:39,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:39,073 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:39,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:39,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/46ab52e8da22432a958f47f0209d4e05 is 50, key is test_row_0/A:col10/1731186998139/Put/seqid=0 2024-11-09T21:16:39,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742070_1246 (size=12001) 2024-11-09T21:16:39,092 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/46ab52e8da22432a958f47f0209d4e05 2024-11-09T21:16:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/95a288c64bfe4dbdbc8cafa572ec9ba4 is 50, key is test_row_0/B:col10/1731186998139/Put/seqid=0 2024-11-09T21:16:39,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742071_1247 (size=12001) 2024-11-09T21:16:39,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:39,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:39,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187059304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,311 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187059310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187059310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,316 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,376 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/5af3a23aac9445f4980b54c28c75908a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5af3a23aac9445f4980b54c28c75908a 2024-11-09T21:16:39,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:39,376 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b3e433ca19674f9e959c2c830df93a6e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b3e433ca19674f9e959c2c830df93a6e 2024-11-09T21:16:39,380 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into b3e433ca19674f9e959c2c830df93a6e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:39,380 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 5af3a23aac9445f4980b54c28c75908a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:39,380 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:39,380 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:39,380 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731186998945; duration=0sec 2024-11-09T21:16:39,380 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731186998946; duration=0sec 2024-11-09T21:16:39,380 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:39,380 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:39,380 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:39,380 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:39,381 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:39,382 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:39,382 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:39,382 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:39,382 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/05b54913b1564db89bbce3ae77105dd0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/46bb94ec4f644fc98ba70fb07ab08c7e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a0048701cf9b459fbe5f72253869a0b8] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.2 K 2024-11-09T21:16:39,383 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 05b54913b1564db89bbce3ae77105dd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731186997235 2024-11-09T21:16:39,383 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 46bb94ec4f644fc98ba70fb07ab08c7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731186997329 2024-11-09T21:16:39,383 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a0048701cf9b459fbe5f72253869a0b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186997960 2024-11-09T21:16:39,395 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#202 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:39,395 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8e34ab30e43646f1b79e0443fbc44b8c is 50, key is test_row_0/C:col10/1731186997961/Put/seqid=0 2024-11-09T21:16:39,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742072_1248 (size=12104) 2024-11-09T21:16:39,413 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8e34ab30e43646f1b79e0443fbc44b8c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8e34ab30e43646f1b79e0443fbc44b8c 2024-11-09T21:16:39,419 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 8e34ab30e43646f1b79e0443fbc44b8c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:39,419 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:39,419 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731186998948; duration=0sec 2024-11-09T21:16:39,419 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:39,419 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:39,420 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187059416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,422 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187059427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187059427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187059496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187059496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,501 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2170 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,502 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,518 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/95a288c64bfe4dbdbc8cafa572ec9ba4 2024-11-09T21:16:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/537529b1979947b68037d41567f6c4bf is 50, key is test_row_0/C:col10/1731186998139/Put/seqid=0 2024-11-09T21:16:39,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742073_1249 (size=12001) 2024-11-09T21:16:39,534 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/537529b1979947b68037d41567f6c4bf 2024-11-09T21:16:39,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/46ab52e8da22432a958f47f0209d4e05 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/46ab52e8da22432a958f47f0209d4e05 2024-11-09T21:16:39,542 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/46ab52e8da22432a958f47f0209d4e05, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:16:39,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/95a288c64bfe4dbdbc8cafa572ec9ba4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/95a288c64bfe4dbdbc8cafa572ec9ba4 2024-11-09T21:16:39,548 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/95a288c64bfe4dbdbc8cafa572ec9ba4, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:16:39,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/537529b1979947b68037d41567f6c4bf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/537529b1979947b68037d41567f6c4bf 2024-11-09T21:16:39,556 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/537529b1979947b68037d41567f6c4bf, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:16:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-09T21:16:39,557 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for c7ac4dbed742a563978c6ff5cf7dc21b in 484ms, sequenceid=97, compaction requested=false 2024-11-09T21:16:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:39,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:39,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-09T21:16:39,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-09T21:16:39,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-09T21:16:39,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3280 sec 2024-11-09T21:16:39,562 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.3350 sec 2024-11-09T21:16:39,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:39,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:16:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:39,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:39,635 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/1c8fb74cef594300a8a71969ece851d1 is 50, key is test_row_0/A:col10/1731186999303/Put/seqid=0 2024-11-09T21:16:39,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742074_1250 (size=14341) 2024-11-09T21:16:39,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/1c8fb74cef594300a8a71969ece851d1 2024-11-09T21:16:39,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/a874f49e3a7445868448ee56455db7dd is 50, key is test_row_0/B:col10/1731186999303/Put/seqid=0 2024-11-09T21:16:39,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742075_1251 (size=12001) 2024-11-09T21:16:39,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/a874f49e3a7445868448ee56455db7dd 2024-11-09T21:16:39,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186999687, max timestamp=1731186999698Num uniques 859; 2024-11-09T21:16:39,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186999687, max timestamp=1731186999698Num uniques 859; 2024-11-09T21:16:39,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731186999687, max timestamp=1731186999698Num uniques 859; 2024-11-09T21:16:39,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:39,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:39,702 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:39,709 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/f63a91634edd4f7791e53e7ef2cfd9ad is 50, key is test_row_0/C:col10/1731186999303/Put/seqid=0 2024-11-09T21:16:39,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742076_1252 (size=12001) 2024-11-09T21:16:39,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187059732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187059735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187059738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,744 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,850 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187059846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187059846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187059846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187060056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187060056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187060056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,061 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=112 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/f63a91634edd4f7791e53e7ef2cfd9ad 2024-11-09T21:16:40,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/1c8fb74cef594300a8a71969ece851d1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/1c8fb74cef594300a8a71969ece851d1 2024-11-09T21:16:40,132 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/1c8fb74cef594300a8a71969ece851d1, entries=200, sequenceid=112, filesize=14.0 K 2024-11-09T21:16:40,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/a874f49e3a7445868448ee56455db7dd as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/a874f49e3a7445868448ee56455db7dd 2024-11-09T21:16:40,136 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/a874f49e3a7445868448ee56455db7dd, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T21:16:40,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/f63a91634edd4f7791e53e7ef2cfd9ad as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/f63a91634edd4f7791e53e7ef2cfd9ad 2024-11-09T21:16:40,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/f63a91634edd4f7791e53e7ef2cfd9ad, entries=150, sequenceid=112, filesize=11.7 K 2024-11-09T21:16:40,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for c7ac4dbed742a563978c6ff5cf7dc21b in 512ms, sequenceid=112, compaction requested=true 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:40,141 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:40,141 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:40,141 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:40,142 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:40,142 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:40,142 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:40,142 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:40,142 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:40,142 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:40,143 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5af3a23aac9445f4980b54c28c75908a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/46ab52e8da22432a958f47f0209d4e05, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/1c8fb74cef594300a8a71969ece851d1] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.5 K 2024-11-09T21:16:40,143 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b3e433ca19674f9e959c2c830df93a6e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/95a288c64bfe4dbdbc8cafa572ec9ba4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/a874f49e3a7445868448ee56455db7dd] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.3 K 2024-11-09T21:16:40,143 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b3e433ca19674f9e959c2c830df93a6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186997960 2024-11-09T21:16:40,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5af3a23aac9445f4980b54c28c75908a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186997960 2024-11-09T21:16:40,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46ab52e8da22432a958f47f0209d4e05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186998139 2024-11-09T21:16:40,143 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 95a288c64bfe4dbdbc8cafa572ec9ba4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186998139 2024-11-09T21:16:40,143 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c8fb74cef594300a8a71969ece851d1, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731186999301 2024-11-09T21:16:40,144 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a874f49e3a7445868448ee56455db7dd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731186999303 2024-11-09T21:16:40,150 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#207 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:40,150 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/c991cab9f5ec4a569d531470b7955716 is 50, key is test_row_0/B:col10/1731186999303/Put/seqid=0 2024-11-09T21:16:40,150 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:40,151 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/627f2d64855544458ec9b6a9e365579c is 50, key is test_row_0/A:col10/1731186999303/Put/seqid=0 2024-11-09T21:16:40,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742077_1253 (size=12207) 2024-11-09T21:16:40,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742078_1254 (size=12207) 2024-11-09T21:16:40,197 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:16:40,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:40,372 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-09T21:16:40,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:40,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:40,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:40,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:40,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:40,373 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:40,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3df8470c0bb341208ca821d8b8de3c92 is 50, key is test_row_0/A:col10/1731186999734/Put/seqid=0 2024-11-09T21:16:40,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742079_1255 (size=16931) 2024-11-09T21:16:40,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187060393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187060393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187060394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187060507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187060507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187060507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,571 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/c991cab9f5ec4a569d531470b7955716 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/c991cab9f5ec4a569d531470b7955716 2024-11-09T21:16:40,580 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/627f2d64855544458ec9b6a9e365579c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/627f2d64855544458ec9b6a9e365579c 2024-11-09T21:16:40,583 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into c991cab9f5ec4a569d531470b7955716(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:40,583 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:40,583 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187000141; duration=0sec 2024-11-09T21:16:40,584 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:40,584 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:40,584 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:40,588 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:40,589 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:40,589 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:40,589 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8e34ab30e43646f1b79e0443fbc44b8c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/537529b1979947b68037d41567f6c4bf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/f63a91634edd4f7791e53e7ef2cfd9ad] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.3 K 2024-11-09T21:16:40,591 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 627f2d64855544458ec9b6a9e365579c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:40,592 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:40,592 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187000141; duration=0sec 2024-11-09T21:16:40,592 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:40,592 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:40,592 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e34ab30e43646f1b79e0443fbc44b8c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731186997960 2024-11-09T21:16:40,593 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 537529b1979947b68037d41567f6c4bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731186998139 2024-11-09T21:16:40,593 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f63a91634edd4f7791e53e7ef2cfd9ad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731186999303 2024-11-09T21:16:40,602 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:40,602 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/9979ebb76bc44b958156b75a2ce9db7f is 50, key is test_row_0/C:col10/1731186999303/Put/seqid=0 2024-11-09T21:16:40,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742080_1256 (size=12207) 2024-11-09T21:16:40,638 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/9979ebb76bc44b958156b75a2ce9db7f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9979ebb76bc44b958156b75a2ce9db7f 2024-11-09T21:16:40,645 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 9979ebb76bc44b958156b75a2ce9db7f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:40,645 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:40,645 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187000141; duration=0sec 2024-11-09T21:16:40,645 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:40,645 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187060717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187060717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187060717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:40,721 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:40,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3df8470c0bb341208ca821d8b8de3c92 2024-11-09T21:16:40,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/400493f7d9404ff38f2eefc76c1f8dac is 50, key is test_row_0/B:col10/1731186999734/Put/seqid=0 2024-11-09T21:16:40,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742081_1257 (size=12151) 2024-11-09T21:16:40,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/400493f7d9404ff38f2eefc76c1f8dac 2024-11-09T21:16:40,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8c1ab557fc57456b8e91b704e00d9828 is 50, key is test_row_0/C:col10/1731186999734/Put/seqid=0 2024-11-09T21:16:40,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742082_1258 (size=12151) 2024-11-09T21:16:40,832 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=149 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8c1ab557fc57456b8e91b704e00d9828 2024-11-09T21:16:40,837 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3df8470c0bb341208ca821d8b8de3c92 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3df8470c0bb341208ca821d8b8de3c92 2024-11-09T21:16:40,841 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3df8470c0bb341208ca821d8b8de3c92, entries=250, sequenceid=149, filesize=16.5 K 2024-11-09T21:16:40,842 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/400493f7d9404ff38f2eefc76c1f8dac as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/400493f7d9404ff38f2eefc76c1f8dac 2024-11-09T21:16:40,845 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/400493f7d9404ff38f2eefc76c1f8dac, entries=150, sequenceid=149, filesize=11.9 K 2024-11-09T21:16:40,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8c1ab557fc57456b8e91b704e00d9828 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8c1ab557fc57456b8e91b704e00d9828 2024-11-09T21:16:40,852 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8c1ab557fc57456b8e91b704e00d9828, entries=150, sequenceid=149, filesize=11.9 K 2024-11-09T21:16:40,852 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for c7ac4dbed742a563978c6ff5cf7dc21b in 480ms, sequenceid=149, compaction requested=false 2024-11-09T21:16:40,852 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:41,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:41,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:16:41,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:41,032 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:41,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/daceb7acecd6431da788ff5191f41d13 is 50, key is test_row_0/A:col10/1731187001029/Put/seqid=0 2024-11-09T21:16:41,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742083_1259 (size=12151) 2024-11-09T21:16:41,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187061063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187061069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187061073, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187061176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,179 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187061176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,190 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187061187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,190 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,191 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-09T21:16:41,387 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:41,387 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:41,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-09T21:16:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187061387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,389 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,389 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:41,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187061387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T21:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,390 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,392 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:41,392 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:41,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187061396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/daceb7acecd6431da788ff5191f41d13 2024-11-09T21:16:41,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/3af8b24f47344f37b752c79847e33caf is 50, key is test_row_0/B:col10/1731187001029/Put/seqid=0 2024-11-09T21:16:41,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742084_1260 (size=12151) 2024-11-09T21:16:41,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T21:16:41,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187061516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,520 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,520 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187061526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,529 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,553 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T21:16:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:41,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187061696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187061696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T21:16:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T21:16:41,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:41,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187061706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:41,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:41,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T21:16:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:41,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/3af8b24f47344f37b752c79847e33caf 2024-11-09T21:16:41,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/54027cdfab8f40ce8e988ec50acee4db is 50, key is test_row_0/C:col10/1731187001029/Put/seqid=0 2024-11-09T21:16:41,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742085_1261 (size=12151) 2024-11-09T21:16:41,891 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=164 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/54027cdfab8f40ce8e988ec50acee4db 2024-11-09T21:16:41,895 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/daceb7acecd6431da788ff5191f41d13 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/daceb7acecd6431da788ff5191f41d13 2024-11-09T21:16:41,898 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/daceb7acecd6431da788ff5191f41d13, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T21:16:41,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/3af8b24f47344f37b752c79847e33caf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/3af8b24f47344f37b752c79847e33caf 2024-11-09T21:16:41,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/3af8b24f47344f37b752c79847e33caf, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T21:16:41,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/54027cdfab8f40ce8e988ec50acee4db as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54027cdfab8f40ce8e988ec50acee4db 2024-11-09T21:16:41,907 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54027cdfab8f40ce8e988ec50acee4db, entries=150, sequenceid=164, filesize=11.9 K 2024-11-09T21:16:41,908 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for c7ac4dbed742a563978c6ff5cf7dc21b in 878ms, sequenceid=164, compaction requested=true 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:41,908 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:41,908 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:41,908 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:41,909 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:41,909 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:41,909 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41289 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:41,909 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,909 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:41,909 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:41,909 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/c991cab9f5ec4a569d531470b7955716, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/400493f7d9404ff38f2eefc76c1f8dac, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/3af8b24f47344f37b752c79847e33caf] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.7 K 2024-11-09T21:16:41,909 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/627f2d64855544458ec9b6a9e365579c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3df8470c0bb341208ca821d8b8de3c92, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/daceb7acecd6431da788ff5191f41d13] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=40.3 K 2024-11-09T21:16:41,910 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 627f2d64855544458ec9b6a9e365579c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731186999303 2024-11-09T21:16:41,910 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c991cab9f5ec4a569d531470b7955716, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731186999303 2024-11-09T21:16:41,910 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 400493f7d9404ff38f2eefc76c1f8dac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731186999734 2024-11-09T21:16:41,910 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3df8470c0bb341208ca821d8b8de3c92, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731186999731 2024-11-09T21:16:41,911 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af8b24f47344f37b752c79847e33caf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731187000381 2024-11-09T21:16:41,911 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting daceb7acecd6431da788ff5191f41d13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731187000381 2024-11-09T21:16:41,918 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#216 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:41,918 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3f1fd686134e4c20a4f28ccb78f09b99 is 50, key is test_row_0/A:col10/1731187001029/Put/seqid=0 2024-11-09T21:16:41,931 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:41,931 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/bb6ed32337294a6a9f7f522a3f100dfb is 50, key is test_row_0/B:col10/1731187001029/Put/seqid=0 2024-11-09T21:16:41,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742086_1262 (size=12459) 2024-11-09T21:16:41,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742087_1263 (size=12459) 2024-11-09T21:16:42,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-09T21:16:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,011 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:42,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:42,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/81877cb88f2c4f16ae30fed83d66ccc2 is 50, key is test_row_0/A:col10/1731187001067/Put/seqid=0 2024-11-09T21:16:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T21:16:42,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742088_1264 (size=12151) 2024-11-09T21:16:42,021 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/81877cb88f2c4f16ae30fed83d66ccc2 2024-11-09T21:16:42,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0acb4191d9464e8bb51da440fdb1f78a is 50, key is test_row_0/B:col10/1731187001067/Put/seqid=0 2024-11-09T21:16:42,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742089_1265 (size=12151) 2024-11-09T21:16:42,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:42,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:42,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187062228, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,236 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187062234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187062235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,241 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,349 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3f1fd686134e4c20a4f28ccb78f09b99 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3f1fd686134e4c20a4f28ccb78f09b99 2024-11-09T21:16:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187062347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187062347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187062347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,352 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,355 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/bb6ed32337294a6a9f7f522a3f100dfb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bb6ed32337294a6a9f7f522a3f100dfb 2024-11-09T21:16:42,356 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 3f1fd686134e4c20a4f28ccb78f09b99(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:42,356 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:42,356 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187001908; duration=0sec 2024-11-09T21:16:42,356 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:42,356 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:42,356 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:42,358 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:42,358 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:42,358 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,358 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9979ebb76bc44b958156b75a2ce9db7f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8c1ab557fc57456b8e91b704e00d9828, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54027cdfab8f40ce8e988ec50acee4db] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.7 K 2024-11-09T21:16:42,359 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9979ebb76bc44b958156b75a2ce9db7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=112, earliestPutTs=1731186999303 2024-11-09T21:16:42,360 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c1ab557fc57456b8e91b704e00d9828, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=149, earliestPutTs=1731186999734 2024-11-09T21:16:42,360 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54027cdfab8f40ce8e988ec50acee4db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731187000381 2024-11-09T21:16:42,362 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into bb6ed32337294a6a9f7f522a3f100dfb(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:42,362 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:42,362 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187001908; duration=0sec 2024-11-09T21:16:42,362 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:42,362 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:42,370 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#220 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:42,370 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/3f461f641c1346f8bf746856b9a7445b is 50, key is test_row_0/C:col10/1731187001029/Put/seqid=0 2024-11-09T21:16:42,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742090_1266 (size=12459) 2024-11-09T21:16:42,451 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0acb4191d9464e8bb51da440fdb1f78a 2024-11-09T21:16:42,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/644f3df39f154f2db952d4afa2bbbd73 is 50, key is test_row_0/C:col10/1731187001067/Put/seqid=0 2024-11-09T21:16:42,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742091_1267 (size=12151) 2024-11-09T21:16:42,465 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/644f3df39f154f2db952d4afa2bbbd73 2024-11-09T21:16:42,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/81877cb88f2c4f16ae30fed83d66ccc2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/81877cb88f2c4f16ae30fed83d66ccc2 2024-11-09T21:16:42,485 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/81877cb88f2c4f16ae30fed83d66ccc2, entries=150, sequenceid=184, filesize=11.9 K 2024-11-09T21:16:42,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0acb4191d9464e8bb51da440fdb1f78a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0acb4191d9464e8bb51da440fdb1f78a 2024-11-09T21:16:42,492 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0acb4191d9464e8bb51da440fdb1f78a, entries=150, sequenceid=184, filesize=11.9 K 2024-11-09T21:16:42,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/644f3df39f154f2db952d4afa2bbbd73 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/644f3df39f154f2db952d4afa2bbbd73 2024-11-09T21:16:42,498 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/644f3df39f154f2db952d4afa2bbbd73, entries=150, sequenceid=184, filesize=11.9 K 2024-11-09T21:16:42,499 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c7ac4dbed742a563978c6ff5cf7dc21b in 488ms, sequenceid=184, compaction requested=false 2024-11-09T21:16:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-09T21:16:42,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-09T21:16:42,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-09T21:16:42,503 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1090 sec 2024-11-09T21:16:42,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 1.1180 sec 2024-11-09T21:16:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-09T21:16:42,526 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:42,526 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-09T21:16:42,528 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:42,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T21:16:42,529 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:42,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:42,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:16:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:42,559 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:42,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/9bc4b82d13ad47e1aeab5f5e55f8c474 is 50, key is test_row_0/A:col10/1731187002231/Put/seqid=0 2024-11-09T21:16:42,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742092_1268 (size=14541) 2024-11-09T21:16:42,579 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/9bc4b82d13ad47e1aeab5f5e55f8c474 2024-11-09T21:16:42,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/795d717f2e674faeaec810c830065e7e is 50, key is test_row_0/B:col10/1731187002231/Put/seqid=0 2024-11-09T21:16:42,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742093_1269 (size=12151) 2024-11-09T21:16:42,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187062599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187062602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187062602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T21:16:42,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T21:16:42,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:42,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,710 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187062706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,710 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187062716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187062716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,791 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/3f461f641c1346f8bf746856b9a7445b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/3f461f641c1346f8bf746856b9a7445b 2024-11-09T21:16:42,795 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 3f461f641c1346f8bf746856b9a7445b(size=12.2 K), total size for store is 24.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:42,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:42,795 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187001908; duration=0sec 2024-11-09T21:16:42,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:42,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:42,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T21:16:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T21:16:42,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187062916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,919 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187062926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,928 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187062926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:42,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:42,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T21:16:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:42,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:42,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/795d717f2e674faeaec810c830065e7e 2024-11-09T21:16:42,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a728c7764b374182a74e27ce64a7eea0 is 50, key is test_row_0/C:col10/1731187002231/Put/seqid=0 2024-11-09T21:16:43,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742094_1270 (size=12151) 2024-11-09T21:16:43,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T21:16:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:43,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:43,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:43,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T21:16:43,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187063226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,228 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187063237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187063237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T21:16:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:43,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:43,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:43,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=199 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a728c7764b374182a74e27ce64a7eea0 2024-11-09T21:16:43,417 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/9bc4b82d13ad47e1aeab5f5e55f8c474 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9bc4b82d13ad47e1aeab5f5e55f8c474 2024-11-09T21:16:43,420 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9bc4b82d13ad47e1aeab5f5e55f8c474, entries=200, sequenceid=199, filesize=14.2 K 2024-11-09T21:16:43,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/795d717f2e674faeaec810c830065e7e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/795d717f2e674faeaec810c830065e7e 2024-11-09T21:16:43,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/795d717f2e674faeaec810c830065e7e, entries=150, sequenceid=199, filesize=11.9 K 2024-11-09T21:16:43,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a728c7764b374182a74e27ce64a7eea0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a728c7764b374182a74e27ce64a7eea0 2024-11-09T21:16:43,429 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a728c7764b374182a74e27ce64a7eea0, entries=150, sequenceid=199, filesize=11.9 K 2024-11-09T21:16:43,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for c7ac4dbed742a563978c6ff5cf7dc21b in 871ms, sequenceid=199, compaction requested=true 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:43,430 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:43,430 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:43,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:43,431 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:43,431 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:43,431 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:43,431 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:43,431 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,431 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,431 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bb6ed32337294a6a9f7f522a3f100dfb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0acb4191d9464e8bb51da440fdb1f78a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/795d717f2e674faeaec810c830065e7e] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.9 K 2024-11-09T21:16:43,431 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3f1fd686134e4c20a4f28ccb78f09b99, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/81877cb88f2c4f16ae30fed83d66ccc2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9bc4b82d13ad47e1aeab5f5e55f8c474] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=38.2 K 2024-11-09T21:16:43,432 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bb6ed32337294a6a9f7f522a3f100dfb, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731187000381 2024-11-09T21:16:43,432 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f1fd686134e4c20a4f28ccb78f09b99, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731187000381 2024-11-09T21:16:43,432 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0acb4191d9464e8bb51da440fdb1f78a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731187001060 2024-11-09T21:16:43,432 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 81877cb88f2c4f16ae30fed83d66ccc2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731187001060 2024-11-09T21:16:43,432 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 795d717f2e674faeaec810c830065e7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731187002231 2024-11-09T21:16:43,432 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc4b82d13ad47e1aeab5f5e55f8c474, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731187002225 2024-11-09T21:16:43,446 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-09T21:16:43,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:16:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:43,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:43,448 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#225 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:43,448 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:43,449 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/49cce14f031845719dd4d15ea5ab56be is 50, key is test_row_0/A:col10/1731187002231/Put/seqid=0 2024-11-09T21:16:43,449 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/abece10ad3a942d88656e9fd1fe66f67 is 50, key is test_row_0/B:col10/1731187002231/Put/seqid=0 2024-11-09T21:16:43,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/f8f5734786764a4aadd9c473c1a84bfe is 50, key is test_row_0/A:col10/1731187002599/Put/seqid=0 2024-11-09T21:16:43,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742095_1271 (size=12561) 2024-11-09T21:16:43,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742096_1272 (size=12561) 2024-11-09T21:16:43,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742097_1273 (size=12151) 2024-11-09T21:16:43,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T21:16:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:43,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187063792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187063792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,800 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187063799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,805 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,866 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/49cce14f031845719dd4d15ea5ab56be as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/49cce14f031845719dd4d15ea5ab56be 2024-11-09T21:16:43,870 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 49cce14f031845719dd4d15ea5ab56be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:43,870 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:43,870 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187003430; duration=0sec 2024-11-09T21:16:43,870 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:43,870 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:43,870 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:43,870 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:43,871 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:43,871 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:43,871 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/3f461f641c1346f8bf746856b9a7445b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/644f3df39f154f2db952d4afa2bbbd73, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a728c7764b374182a74e27ce64a7eea0] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=35.9 K 2024-11-09T21:16:43,871 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3f461f641c1346f8bf746856b9a7445b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=164, earliestPutTs=1731187000381 2024-11-09T21:16:43,871 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 644f3df39f154f2db952d4afa2bbbd73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731187001060 2024-11-09T21:16:43,871 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a728c7764b374182a74e27ce64a7eea0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731187002231 2024-11-09T21:16:43,877 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/f8f5734786764a4aadd9c473c1a84bfe 2024-11-09T21:16:43,878 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#228 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:43,879 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/0959552a6fca4b5babe87c3622c42a87 is 50, key is test_row_0/C:col10/1731187002231/Put/seqid=0 2024-11-09T21:16:43,882 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/abece10ad3a942d88656e9fd1fe66f67 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/abece10ad3a942d88656e9fd1fe66f67 2024-11-09T21:16:43,887 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into abece10ad3a942d88656e9fd1fe66f67(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:43,887 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:43,887 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187003430; duration=0sec 2024-11-09T21:16:43,888 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:43,888 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:43,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d8a424631be14028b437f97ca34d21c8 is 50, key is test_row_0/B:col10/1731187002599/Put/seqid=0 2024-11-09T21:16:43,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742098_1274 (size=12561) 2024-11-09T21:16:43,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742099_1275 (size=12151) 2024-11-09T21:16:43,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187063906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187063906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:43,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:43,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187063907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187064116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187064116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187064116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,299 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/0959552a6fca4b5babe87c3622c42a87 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/0959552a6fca4b5babe87c3622c42a87 2024-11-09T21:16:44,303 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 0959552a6fca4b5babe87c3622c42a87(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:44,303 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:44,303 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187003430; duration=0sec 2024-11-09T21:16:44,303 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:44,303 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:44,306 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d8a424631be14028b437f97ca34d21c8 2024-11-09T21:16:44,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/38f2227bd4774021beca904a975e0161 is 50, key is test_row_0/C:col10/1731187002599/Put/seqid=0 2024-11-09T21:16:44,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742100_1276 (size=12151) 2024-11-09T21:16:44,322 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/38f2227bd4774021beca904a975e0161 2024-11-09T21:16:44,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/f8f5734786764a4aadd9c473c1a84bfe as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f8f5734786764a4aadd9c473c1a84bfe 2024-11-09T21:16:44,332 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f8f5734786764a4aadd9c473c1a84bfe, entries=150, sequenceid=220, filesize=11.9 K 2024-11-09T21:16:44,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d8a424631be14028b437f97ca34d21c8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d8a424631be14028b437f97ca34d21c8 2024-11-09T21:16:44,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d8a424631be14028b437f97ca34d21c8, entries=150, sequenceid=220, filesize=11.9 K 2024-11-09T21:16:44,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/38f2227bd4774021beca904a975e0161 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/38f2227bd4774021beca904a975e0161 2024-11-09T21:16:44,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/38f2227bd4774021beca904a975e0161, entries=150, sequenceid=220, filesize=11.9 K 2024-11-09T21:16:44,343 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for c7ac4dbed742a563978c6ff5cf7dc21b in 897ms, sequenceid=220, compaction requested=false 2024-11-09T21:16:44,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:44,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:44,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-09T21:16:44,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-09T21:16:44,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-09T21:16:44,346 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8150 sec 2024-11-09T21:16:44,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.8200 sec 2024-11-09T21:16:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:44,430 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:16:44,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:44,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:44,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:44,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:44,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b059b85079444fd8bfccb30c210a6f20 is 50, key is test_row_0/A:col10/1731187004428/Put/seqid=0 2024-11-09T21:16:44,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742101_1277 (size=14541) 2024-11-09T21:16:44,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187064456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,462 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,463 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,464 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187064460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,465 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187064460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,465 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187064566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187064566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187064576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-09T21:16:44,677 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:44,677 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-09T21:16:44,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T21:16:44,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:44,679 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:44,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187064776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187064776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,779 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T21:16:44,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:44,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187064786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:44,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:44,831 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T21:16:44,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:44,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:44,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:44,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:44,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:44,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:44,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b059b85079444fd8bfccb30c210a6f20 2024-11-09T21:16:44,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/9e11ff2d0ab14adfa31088821228662a is 50, key is test_row_0/B:col10/1731187004428/Put/seqid=0 2024-11-09T21:16:44,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742102_1278 (size=12151) 2024-11-09T21:16:44,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/9e11ff2d0ab14adfa31088821228662a 2024-11-09T21:16:44,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6202a37e56db4755ab0ec7933cb7e9ad is 50, key is test_row_0/C:col10/1731187004428/Put/seqid=0 2024-11-09T21:16:44,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742103_1279 (size=12151) 2024-11-09T21:16:44,983 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T21:16:44,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:44,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:44,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:44,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:44,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T21:16:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187065086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187065086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,090 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187065096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T21:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,137 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:45,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:45,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:45,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6202a37e56db4755ab0ec7933cb7e9ad 2024-11-09T21:16:45,264 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b059b85079444fd8bfccb30c210a6f20 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b059b85079444fd8bfccb30c210a6f20 2024-11-09T21:16:45,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b059b85079444fd8bfccb30c210a6f20, entries=200, sequenceid=237, filesize=14.2 K 2024-11-09T21:16:45,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/9e11ff2d0ab14adfa31088821228662a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9e11ff2d0ab14adfa31088821228662a 2024-11-09T21:16:45,273 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9e11ff2d0ab14adfa31088821228662a, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T21:16:45,274 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6202a37e56db4755ab0ec7933cb7e9ad as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6202a37e56db4755ab0ec7933cb7e9ad 2024-11-09T21:16:45,277 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6202a37e56db4755ab0ec7933cb7e9ad, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T21:16:45,277 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for c7ac4dbed742a563978c6ff5cf7dc21b in 848ms, sequenceid=237, compaction requested=true 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:45,278 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:45,278 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:45,278 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:45,279 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,279 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,279 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/49cce14f031845719dd4d15ea5ab56be, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f8f5734786764a4aadd9c473c1a84bfe, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b059b85079444fd8bfccb30c210a6f20] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=38.3 K 2024-11-09T21:16:45,279 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/abece10ad3a942d88656e9fd1fe66f67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d8a424631be14028b437f97ca34d21c8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9e11ff2d0ab14adfa31088821228662a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=36.0 K 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting abece10ad3a942d88656e9fd1fe66f67, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731187002231 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cce14f031845719dd4d15ea5ab56be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731187002231 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting d8a424631be14028b437f97ca34d21c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731187002587 2024-11-09T21:16:45,279 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f5734786764a4aadd9c473c1a84bfe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731187002587 2024-11-09T21:16:45,280 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e11ff2d0ab14adfa31088821228662a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187003798 2024-11-09T21:16:45,280 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b059b85079444fd8bfccb30c210a6f20, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187003756 2024-11-09T21:16:45,286 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:45,287 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/ae0621505a754ea18941234521a57599 is 50, key is test_row_0/B:col10/1731187004428/Put/seqid=0 2024-11-09T21:16:45,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,290 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:45,290 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:45,290 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b1dd3d74ee4c4b128d6ebb60ce02e229 is 50, key is test_row_0/A:col10/1731187004428/Put/seqid=0 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:45,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:45,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/965246391be943d2b31fb61b478ebfbc is 50, key is test_row_0/A:col10/1731187004458/Put/seqid=0 2024-11-09T21:16:45,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742105_1281 (size=12663) 2024-11-09T21:16:45,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742106_1282 (size=12151) 2024-11-09T21:16:45,302 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/965246391be943d2b31fb61b478ebfbc 2024-11-09T21:16:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T21:16:45,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742104_1280 (size=12663) 2024-11-09T21:16:45,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/86c703edca644aa7ba3d11ae8d34bd5d is 50, key is test_row_0/B:col10/1731187004458/Put/seqid=0 2024-11-09T21:16:45,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742107_1283 (size=12151) 2024-11-09T21:16:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:45,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:45,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187065587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187065588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187065597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187065597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187065606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,699 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b1dd3d74ee4c4b128d6ebb60ce02e229 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b1dd3d74ee4c4b128d6ebb60ce02e229 2024-11-09T21:16:45,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187065696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187065696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,704 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into b1dd3d74ee4c4b128d6ebb60ce02e229(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:45,704 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:45,705 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187005278; duration=0sec 2024-11-09T21:16:45,705 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:45,705 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:45,705 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:45,706 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:45,706 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:45,706 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,706 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/0959552a6fca4b5babe87c3622c42a87, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/38f2227bd4774021beca904a975e0161, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6202a37e56db4755ab0ec7933cb7e9ad] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=36.0 K 2024-11-09T21:16:45,706 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0959552a6fca4b5babe87c3622c42a87, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=199, earliestPutTs=1731187002231 2024-11-09T21:16:45,707 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38f2227bd4774021beca904a975e0161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731187002587 2024-11-09T21:16:45,707 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6202a37e56db4755ab0ec7933cb7e9ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187003798 2024-11-09T21:16:45,713 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:45,714 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/798f64b0649446fabda49cf7e4b36d6a is 50, key is test_row_0/C:col10/1731187004428/Put/seqid=0 2024-11-09T21:16:45,715 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/ae0621505a754ea18941234521a57599 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/ae0621505a754ea18941234521a57599 2024-11-09T21:16:45,720 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into ae0621505a754ea18941234521a57599(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:45,720 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:45,720 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187005278; duration=0sec 2024-11-09T21:16:45,720 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:45,720 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:45,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742108_1284 (size=12663) 2024-11-09T21:16:45,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/86c703edca644aa7ba3d11ae8d34bd5d 2024-11-09T21:16:45,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6dee2330cf83435ebf7ef20270634eeb is 50, key is test_row_0/C:col10/1731187004458/Put/seqid=0 2024-11-09T21:16:45,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742109_1285 (size=12151) 2024-11-09T21:16:45,743 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6dee2330cf83435ebf7ef20270634eeb 2024-11-09T21:16:45,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/965246391be943d2b31fb61b478ebfbc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/965246391be943d2b31fb61b478ebfbc 2024-11-09T21:16:45,752 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/965246391be943d2b31fb61b478ebfbc, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T21:16:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/86c703edca644aa7ba3d11ae8d34bd5d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/86c703edca644aa7ba3d11ae8d34bd5d 2024-11-09T21:16:45,758 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/86c703edca644aa7ba3d11ae8d34bd5d, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T21:16:45,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6dee2330cf83435ebf7ef20270634eeb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6dee2330cf83435ebf7ef20270634eeb 2024-11-09T21:16:45,763 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6dee2330cf83435ebf7ef20270634eeb, entries=150, sequenceid=255, filesize=11.9 K 2024-11-09T21:16:45,764 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for c7ac4dbed742a563978c6ff5cf7dc21b in 474ms, sequenceid=255, compaction requested=false 2024-11-09T21:16:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-09T21:16:45,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-09T21:16:45,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-09T21:16:45,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0870 sec 2024-11-09T21:16:45,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.0910 sec 2024-11-09T21:16:45,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-09T21:16:45,817 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:45,817 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:45,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-09T21:16:45,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:45,819 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:45,819 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:45,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:45,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:45,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:16:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:45,909 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:45,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/ffa53238958247c8a6c65c9ef7044e2e is 50, key is test_row_0/A:col10/1731187005587/Put/seqid=0 2024-11-09T21:16:45,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742110_1286 (size=12301) 2024-11-09T21:16:45,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:45,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187065944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,950 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,951 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:45,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187065947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:45,951 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:45,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:45,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:45,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:45,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187066056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187066056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:46,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:46,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,129 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/798f64b0649446fabda49cf7e4b36d6a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/798f64b0649446fabda49cf7e4b36d6a 2024-11-09T21:16:46,133 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 798f64b0649446fabda49cf7e4b36d6a(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:46,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:46,133 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187005278; duration=0sec 2024-11-09T21:16:46,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:46,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:46,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187066267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187066267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,321 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/ffa53238958247c8a6c65c9ef7044e2e 2024-11-09T21:16:46,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b79f0f6b4bdc482ea89b9a44f4d13ed5 is 50, key is test_row_0/B:col10/1731187005587/Put/seqid=0 2024-11-09T21:16:46,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742111_1287 (size=12301) 2024-11-09T21:16:46,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:46,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:46,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187066576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187066576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,580 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:46,584 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,584 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187066616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187066617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,620 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:46,620 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187066626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:46,629 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:46,630 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:46,732 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b79f0f6b4bdc482ea89b9a44f4d13ed5 2024-11-09T21:16:46,737 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5fc116322578478f92df8decaac2e55c is 50, key is test_row_0/C:col10/1731187005587/Put/seqid=0 2024-11-09T21:16:46,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742112_1288 (size=12301) 2024-11-09T21:16:46,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:46,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:46,890 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:47,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:47,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:47,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:47,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:47,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:47,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:47,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:47,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187067086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:47,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:47,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187067086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:47,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:47,144 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5fc116322578478f92df8decaac2e55c 2024-11-09T21:16:47,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/ffa53238958247c8a6c65c9ef7044e2e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/ffa53238958247c8a6c65c9ef7044e2e 2024-11-09T21:16:47,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/ffa53238958247c8a6c65c9ef7044e2e, entries=150, sequenceid=272, filesize=12.0 K 2024-11-09T21:16:47,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b79f0f6b4bdc482ea89b9a44f4d13ed5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b79f0f6b4bdc482ea89b9a44f4d13ed5 2024-11-09T21:16:47,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b79f0f6b4bdc482ea89b9a44f4d13ed5, entries=150, sequenceid=272, filesize=12.0 K 2024-11-09T21:16:47,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5fc116322578478f92df8decaac2e55c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fc116322578478f92df8decaac2e55c 2024-11-09T21:16:47,159 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fc116322578478f92df8decaac2e55c, entries=150, sequenceid=272, filesize=12.0 K 2024-11-09T21:16:47,160 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for c7ac4dbed742a563978c6ff5cf7dc21b in 1251ms, sequenceid=272, compaction requested=true 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:47,160 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:47,160 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:47,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:47,161 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:47,161 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:47,161 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:47,161 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:47,161 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:47,161 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:47,161 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b1dd3d74ee4c4b128d6ebb60ce02e229, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/965246391be943d2b31fb61b478ebfbc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/ffa53238958247c8a6c65c9ef7044e2e] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=36.2 K 2024-11-09T21:16:47,161 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/ae0621505a754ea18941234521a57599, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/86c703edca644aa7ba3d11ae8d34bd5d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b79f0f6b4bdc482ea89b9a44f4d13ed5] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=36.2 K 2024-11-09T21:16:47,162 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ae0621505a754ea18941234521a57599, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187003798 2024-11-09T21:16:47,162 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1dd3d74ee4c4b128d6ebb60ce02e229, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187003798 2024-11-09T21:16:47,168 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965246391be943d2b31fb61b478ebfbc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731187004453 2024-11-09T21:16:47,168 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 86c703edca644aa7ba3d11ae8d34bd5d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731187004453 2024-11-09T21:16:47,168 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b79f0f6b4bdc482ea89b9a44f4d13ed5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731187005567 2024-11-09T21:16:47,168 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ffa53238958247c8a6c65c9ef7044e2e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731187005567 2024-11-09T21:16:47,175 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:47,175 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/bdbd4077d46b40c8a86d571126cecf8f is 50, key is test_row_0/B:col10/1731187005587/Put/seqid=0 2024-11-09T21:16:47,178 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#244 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:47,178 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/503b7f3b99aa40c9846269054db7ac3a is 50, key is test_row_0/A:col10/1731187005587/Put/seqid=0 2024-11-09T21:16:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742113_1289 (size=12915) 2024-11-09T21:16:47,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742114_1290 (size=12915) 2024-11-09T21:16:47,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:47,196 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:47,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:47,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/a5b8b40851ef4d869b87e0e47b5f1542 is 50, key is test_row_0/A:col10/1731187005940/Put/seqid=0 2024-11-09T21:16:47,207 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/503b7f3b99aa40c9846269054db7ac3a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/503b7f3b99aa40c9846269054db7ac3a 2024-11-09T21:16:47,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742115_1291 (size=12301) 2024-11-09T21:16:47,218 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 503b7f3b99aa40c9846269054db7ac3a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:47,218 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:47,218 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187007160; duration=0sec 2024-11-09T21:16:47,218 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:47,218 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:47,218 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:47,219 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:47,220 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:47,220 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:47,220 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/798f64b0649446fabda49cf7e4b36d6a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6dee2330cf83435ebf7ef20270634eeb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fc116322578478f92df8decaac2e55c] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=36.2 K 2024-11-09T21:16:47,220 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 798f64b0649446fabda49cf7e4b36d6a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187003798 2024-11-09T21:16:47,220 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dee2330cf83435ebf7ef20270634eeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=255, earliestPutTs=1731187004453 2024-11-09T21:16:47,220 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fc116322578478f92df8decaac2e55c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731187005567 2024-11-09T21:16:47,227 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#246 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:47,228 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/d6390ca6eed44d21b089b2101f0b00e2 is 50, key is test_row_0/C:col10/1731187005587/Put/seqid=0 2024-11-09T21:16:47,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742116_1292 (size=12915) 2024-11-09T21:16:47,593 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/bdbd4077d46b40c8a86d571126cecf8f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bdbd4077d46b40c8a86d571126cecf8f 2024-11-09T21:16:47,597 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into bdbd4077d46b40c8a86d571126cecf8f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:47,597 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:47,597 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187007160; duration=0sec 2024-11-09T21:16:47,597 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:47,597 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:47,609 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/a5b8b40851ef4d869b87e0e47b5f1542 2024-11-09T21:16:47,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/59bb252b823745ff9b1b0e10f65bb5ad is 50, key is test_row_0/B:col10/1731187005940/Put/seqid=0 2024-11-09T21:16:47,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742117_1293 (size=12301) 2024-11-09T21:16:47,624 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/59bb252b823745ff9b1b0e10f65bb5ad 2024-11-09T21:16:47,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/7b34abcd428d495a8a03249362988722 is 50, key is test_row_0/C:col10/1731187005940/Put/seqid=0 2024-11-09T21:16:47,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742118_1294 (size=12301) 2024-11-09T21:16:47,639 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/d6390ca6eed44d21b089b2101f0b00e2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/d6390ca6eed44d21b089b2101f0b00e2 2024-11-09T21:16:47,643 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into d6390ca6eed44d21b089b2101f0b00e2(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:47,643 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:47,644 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187007160; duration=0sec 2024-11-09T21:16:47,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:47,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:47,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:48,035 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/7b34abcd428d495a8a03249362988722 2024-11-09T21:16:48,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/a5b8b40851ef4d869b87e0e47b5f1542 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/a5b8b40851ef4d869b87e0e47b5f1542 2024-11-09T21:16:48,042 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/a5b8b40851ef4d869b87e0e47b5f1542, entries=150, sequenceid=290, filesize=12.0 K 2024-11-09T21:16:48,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/59bb252b823745ff9b1b0e10f65bb5ad as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/59bb252b823745ff9b1b0e10f65bb5ad 2024-11-09T21:16:48,046 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/59bb252b823745ff9b1b0e10f65bb5ad, entries=150, sequenceid=290, filesize=12.0 K 2024-11-09T21:16:48,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/7b34abcd428d495a8a03249362988722 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/7b34abcd428d495a8a03249362988722 2024-11-09T21:16:48,051 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/7b34abcd428d495a8a03249362988722, entries=150, sequenceid=290, filesize=12.0 K 2024-11-09T21:16:48,052 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=0 B/0 for c7ac4dbed742a563978c6ff5cf7dc21b in 856ms, sequenceid=290, compaction requested=false 2024-11-09T21:16:48,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:48,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:48,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-09T21:16:48,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-09T21:16:48,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-09T21:16:48,055 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2340 sec 2024-11-09T21:16:48,057 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2390 sec 2024-11-09T21:16:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:48,117 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:16:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:48,117 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:48,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/97428cfa74e34619a55ffc85508a2bed is 50, key is test_row_0/A:col10/1731187008110/Put/seqid=0 2024-11-09T21:16:48,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742119_1295 (size=14737) 2024-11-09T21:16:48,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/97428cfa74e34619a55ffc85508a2bed 2024-11-09T21:16:48,138 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0364ec03c91f4be4881de5769618e316 is 50, key is test_row_0/B:col10/1731187008110/Put/seqid=0 2024-11-09T21:16:48,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742120_1296 (size=9857) 2024-11-09T21:16:48,146 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0364ec03c91f4be4881de5769618e316 2024-11-09T21:16:48,153 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/ac524e2cf32e43ae870068a0779a0fce is 50, key is test_row_0/C:col10/1731187008110/Put/seqid=0 2024-11-09T21:16:48,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742121_1297 (size=9857) 2024-11-09T21:16:48,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187008150, max timestamp=1731187008164Num uniques 859; 2024-11-09T21:16:48,166 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:48,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187008150, max timestamp=1731187008164Num uniques 859; 2024-11-09T21:16:48,167 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:48,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187008150, max timestamp=1731187008164Num uniques 859; 2024-11-09T21:16:48,168 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:48,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187068200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,206 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187068206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,210 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187068316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187068317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187068526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187068526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,557 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=303 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/ac524e2cf32e43ae870068a0779a0fce 2024-11-09T21:16:48,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/97428cfa74e34619a55ffc85508a2bed as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/97428cfa74e34619a55ffc85508a2bed 2024-11-09T21:16:48,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/97428cfa74e34619a55ffc85508a2bed, entries=200, sequenceid=303, filesize=14.4 K 2024-11-09T21:16:48,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/0364ec03c91f4be4881de5769618e316 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0364ec03c91f4be4881de5769618e316 2024-11-09T21:16:48,568 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0364ec03c91f4be4881de5769618e316, entries=100, sequenceid=303, filesize=9.6 K 2024-11-09T21:16:48,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/ac524e2cf32e43ae870068a0779a0fce as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ac524e2cf32e43ae870068a0779a0fce 2024-11-09T21:16:48,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ac524e2cf32e43ae870068a0779a0fce, entries=100, sequenceid=303, filesize=9.6 K 2024-11-09T21:16:48,573 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.94 KB/135110 for c7ac4dbed742a563978c6ff5cf7dc21b in 457ms, sequenceid=303, compaction requested=true 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:48,573 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:48,573 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:48,573 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39953 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:48,574 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:48,574 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:48,574 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/503b7f3b99aa40c9846269054db7ac3a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/a5b8b40851ef4d869b87e0e47b5f1542, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/97428cfa74e34619a55ffc85508a2bed] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=39.0 K 2024-11-09T21:16:48,574 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bdbd4077d46b40c8a86d571126cecf8f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/59bb252b823745ff9b1b0e10f65bb5ad, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0364ec03c91f4be4881de5769618e316] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=34.3 K 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503b7f3b99aa40c9846269054db7ac3a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731187005567 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bdbd4077d46b40c8a86d571126cecf8f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731187005567 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5b8b40851ef4d869b87e0e47b5f1542, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731187005940 2024-11-09T21:16:48,574 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 59bb252b823745ff9b1b0e10f65bb5ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731187005940 2024-11-09T21:16:48,575 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97428cfa74e34619a55ffc85508a2bed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731187008110 2024-11-09T21:16:48,575 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0364ec03c91f4be4881de5769618e316, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731187008110 2024-11-09T21:16:48,582 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#252 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:48,582 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/11e3f5f8e2fe4306b75460ca4b96ff3b is 50, key is test_row_0/B:col10/1731187008110/Put/seqid=0 2024-11-09T21:16:48,582 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#253 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:48,583 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/7ffd9baff3ab471baff77d1166822f23 is 50, key is test_row_0/A:col10/1731187008110/Put/seqid=0 2024-11-09T21:16:48,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742122_1298 (size=13017) 2024-11-09T21:16:48,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742123_1299 (size=13017) 2024-11-09T21:16:48,590 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/7ffd9baff3ab471baff77d1166822f23 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7ffd9baff3ab471baff77d1166822f23 2024-11-09T21:16:48,594 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 7ffd9baff3ab471baff77d1166822f23(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:48,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:48,594 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187008573; duration=0sec 2024-11-09T21:16:48,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:48,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:48,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:48,595 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:48,595 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:48,595 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:48,595 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/d6390ca6eed44d21b089b2101f0b00e2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/7b34abcd428d495a8a03249362988722, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ac524e2cf32e43ae870068a0779a0fce] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=34.3 K 2024-11-09T21:16:48,595 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6390ca6eed44d21b089b2101f0b00e2, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731187005567 2024-11-09T21:16:48,595 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b34abcd428d495a8a03249362988722, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1731187005940 2024-11-09T21:16:48,596 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ac524e2cf32e43ae870068a0779a0fce, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731187008110 2024-11-09T21:16:48,601 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#254 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:48,601 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5fbd8098fcb0430e95699f28eebb89f1 is 50, key is test_row_0/C:col10/1731187008110/Put/seqid=0 2024-11-09T21:16:48,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742124_1300 (size=13017) 2024-11-09T21:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:48,640 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=145.36 KB heapSize=415.77 KB 2024-11-09T21:16:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:16:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:48,640 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:16:48,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:48,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:16:48,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/f1a748b482904aa296b2e51d81dda854 is 50, key is test_row_0/A:col10/1731187008639/Put/seqid=0 2024-11-09T21:16:48,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742125_1301 (size=17181) 2024-11-09T21:16:48,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187068649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,653 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,654 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187068653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187068653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187068756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,770 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187068766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187068766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,771 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,838 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187068836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187068836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,839 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187068966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,969 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187068976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187068977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:48,992 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/11e3f5f8e2fe4306b75460ca4b96ff3b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/11e3f5f8e2fe4306b75460ca4b96ff3b 2024-11-09T21:16:48,996 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into 11e3f5f8e2fe4306b75460ca4b96ff3b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:48,996 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:48,996 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187008573; duration=0sec 2024-11-09T21:16:48,996 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:48,996 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:49,010 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5fbd8098fcb0430e95699f28eebb89f1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fbd8098fcb0430e95699f28eebb89f1 2024-11-09T21:16:49,014 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 5fbd8098fcb0430e95699f28eebb89f1(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:49,014 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:49,014 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187008573; duration=0sec 2024-11-09T21:16:49,014 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:49,014 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:49,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/f1a748b482904aa296b2e51d81dda854 2024-11-09T21:16:49,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/23311cef913d4f378395866d67fab26c is 50, key is test_row_0/B:col10/1731187008639/Put/seqid=0 2024-11-09T21:16:49,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742126_1302 (size=12301) 2024-11-09T21:16:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187069276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187069286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187069286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187069346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187069346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/23311cef913d4f378395866d67fab26c 2024-11-09T21:16:49,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8110cc189592493d8a8ef7ef80692fae is 50, key is test_row_0/C:col10/1731187008639/Put/seqid=0 2024-11-09T21:16:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742127_1303 (size=12301) 2024-11-09T21:16:49,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187069786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,789 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187069796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,799 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187069796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:49,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:49,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8110cc189592493d8a8ef7ef80692fae 2024-11-09T21:16:49,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/f1a748b482904aa296b2e51d81dda854 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f1a748b482904aa296b2e51d81dda854 2024-11-09T21:16:49,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f1a748b482904aa296b2e51d81dda854, entries=250, sequenceid=342, filesize=16.8 K 2024-11-09T21:16:49,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/23311cef913d4f378395866d67fab26c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/23311cef913d4f378395866d67fab26c 2024-11-09T21:16:49,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/23311cef913d4f378395866d67fab26c, entries=150, sequenceid=342, filesize=12.0 K 2024-11-09T21:16:49,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/8110cc189592493d8a8ef7ef80692fae as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8110cc189592493d8a8ef7ef80692fae 2024-11-09T21:16:49,893 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8110cc189592493d8a8ef7ef80692fae, entries=150, sequenceid=342, filesize=12.0 K 2024-11-09T21:16:49,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~145.36 KB/148850, heapSize ~415.72 KB/425696, currentSize=33.54 KB/34350 for c7ac4dbed742a563978c6ff5cf7dc21b in 1253ms, sequenceid=342, compaction requested=false 2024-11-09T21:16:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-09T21:16:49,976 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:49,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-09T21:16:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T21:16:49,978 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:49,979 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:49,979 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:50,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T21:16:50,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-09T21:16:50,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:50,131 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T21:16:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:50,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:50,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/70b9aee494874cbd82c629488ceb572f is 50, key is test_row_0/A:col10/1731187008652/Put/seqid=0 2024-11-09T21:16:50,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742128_1304 (size=9857) 2024-11-09T21:16:50,140 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/70b9aee494874cbd82c629488ceb572f 2024-11-09T21:16:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d583e47449484145926ba5657af4874c is 50, key is test_row_0/B:col10/1731187008652/Put/seqid=0 2024-11-09T21:16:50,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742129_1305 (size=9857) 2024-11-09T21:16:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T21:16:50,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:50,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:50,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187010389, max timestamp=1731187010405Num uniques 859; 2024-11-09T21:16:50,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187010389, max timestamp=1731187010405Num uniques 859; 2024-11-09T21:16:50,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:50,408 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:50,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187010389, max timestamp=1731187010405Num uniques 859; 2024-11-09T21:16:50,410 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:50,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187010437, max timestamp=1731187010454Num uniques 1018; 2024-11-09T21:16:50,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187010437, max timestamp=1731187010454Num uniques 1018; 2024-11-09T21:16:50,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187010437, max timestamp=1731187010454Num uniques 1018; 2024-11-09T21:16:50,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:16:50,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:16:50,458 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-09T21:16:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731187010474, max timestamp=1731187010495Num uniques 1027; 2024-11-09T21:16:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T21:16:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731187010474, max timestamp=1731187010495Num uniques 1027; 2024-11-09T21:16:50,499 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T21:16:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731187010474, max timestamp=1731187010495Num uniques 1027; 2024-11-09T21:16:50,502 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T21:16:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187010530, max timestamp=1731187010542Num uniques 1036; 2024-11-09T21:16:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187010530, max timestamp=1731187010542Num uniques 1036; 2024-11-09T21:16:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187010530, max timestamp=1731187010542Num uniques 1036; 2024-11-09T21:16:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T21:16:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T21:16:50,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T21:16:50,551 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d583e47449484145926ba5657af4874c 2024-11-09T21:16:50,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/bbedc8600e3c49bc943b8b2c62d13f04 is 50, key is test_row_0/C:col10/1731187008652/Put/seqid=0 2024-11-09T21:16:50,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742130_1306 (size=9857) 2024-11-09T21:16:50,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731187010577, max timestamp=1731187010597Num uniques 1095; 2024-11-09T21:16:50,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:50,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731187010577, max timestamp=1731187010597Num uniques 1095; 2024-11-09T21:16:50,601 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:50,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731187010585, max timestamp=1731187010597Num uniques 1095; 2024-11-09T21:16:50,603 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=8930, suffix heap size=141848,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=195 2024-11-09T21:16:50,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T21:16:50,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731187010641, max timestamp=1731187010648Num uniques 1104; 2024-11-09T21:16:50,651 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-09T21:16:50,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731187010641, max timestamp=1731187010648Num uniques 1104; 2024-11-09T21:16:50,652 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-09T21:16:50,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731187010641, max timestamp=1731187010648Num uniques 1054; 2024-11-09T21:16:50,655 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-09T21:16:50,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731187010687, max timestamp=1731187010701Num uniques 1013; 2024-11-09T21:16:50,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731187010687, max timestamp=1731187010701Num uniques 1013; 2024-11-09T21:16:50,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T21:16:50,704 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T21:16:50,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731187010687, max timestamp=1731187010701Num uniques 1013; 2024-11-09T21:16:50,705 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-09T21:16:50,753 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:50,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187070747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:50,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731187010718, max timestamp=1731187010749Num uniques 1072; 2024-11-09T21:16:50,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731187010718, max timestamp=1731187010749Num uniques 1072; 2024-11-09T21:16:50,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-09T21:16:50,753 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-09T21:16:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,754 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:50,754 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731187010718, max timestamp=1731187010749Num uniques 1072; 2024-11-09T21:16:50,755 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-09T21:16:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:50,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187070816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:50,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:50,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187070816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:50,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731187010807, max timestamp=1731187010817Num uniques 1031; 2024-11-09T21:16:50,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-09T21:16:50,820 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:50,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731187010807, max timestamp=1731187010817Num uniques 1031; 2024-11-09T21:16:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731187010807, max timestamp=1731187010817Num uniques 1031; 2024-11-09T21:16:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,821 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:50,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-09T21:16:50,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-09T21:16:50,822 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:50,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731187010833, max timestamp=1731187010859Num uniques 1040; 2024-11-09T21:16:50,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-09T21:16:50,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731187010833, max timestamp=1731187010859Num uniques 1040; 2024-11-09T21:16:50,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731187010833, max timestamp=1731187010859Num uniques 1040; 2024-11-09T21:16:50,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-09T21:16:50,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=10990, suffix heap size=136128,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=240 2024-11-09T21:16:50,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:50,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187070889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,893 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:50,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731187010871, max timestamp=1731187010890Num uniques 1099; 2024-11-09T21:16:50,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-09T21:16:50,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731187010871, max timestamp=1731187010890Num uniques 1099; 2024-11-09T21:16:50,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=249, cellSize=11404, totalHeapSize=26136, min timestamp=1731187010871, max timestamp=1731187010890Num uniques 1099; 2024-11-09T21:16:50,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-09T21:16:50,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=11404, suffix heap size=142264,new segment heap size=26136 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=249 2024-11-09T21:16:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:50,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187070939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:50,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:50,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187070941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:50,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:50,965 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=352 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/bbedc8600e3c49bc943b8b2c62d13f04 2024-11-09T21:16:50,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/70b9aee494874cbd82c629488ceb572f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/70b9aee494874cbd82c629488ceb572f 2024-11-09T21:16:50,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/70b9aee494874cbd82c629488ceb572f, entries=100, sequenceid=352, filesize=9.6 K 2024-11-09T21:16:50,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/d583e47449484145926ba5657af4874c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d583e47449484145926ba5657af4874c 2024-11-09T21:16:50,976 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d583e47449484145926ba5657af4874c, entries=100, sequenceid=352, filesize=9.6 K 2024-11-09T21:16:50,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/bbedc8600e3c49bc943b8b2c62d13f04 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/bbedc8600e3c49bc943b8b2c62d13f04 2024-11-09T21:16:50,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/bbedc8600e3c49bc943b8b2c62d13f04, entries=100, sequenceid=352, filesize=9.6 K 2024-11-09T21:16:50,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=147.60 KB/151140 for c7ac4dbed742a563978c6ff5cf7dc21b in 849ms, sequenceid=352, compaction requested=true 2024-11-09T21:16:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:50,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-09T21:16:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-09T21:16:50,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-09T21:16:50,983 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-11-09T21:16:50,985 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0070 sec 2024-11-09T21:16:50,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=154.31 KB heapSize=437.02 KB 2024-11-09T21:16:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:16:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:16:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:51,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-09T21:16:51,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:51,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/50be73876534449998df198303e1b935 is 50, key is test_row_0/A:col10/1731187010998/Put/seqid=0 2024-11-09T21:16:51,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742131_1307 (size=14741) 2024-11-09T21:16:51,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187071020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,024 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187071047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,051 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,058 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187071056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-09T21:16:51,117 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:51,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:51,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-09T21:16:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T21:16:51,119 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:51,120 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:51,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187071127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T21:16:51,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187071256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187071266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,269 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T21:16:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:51,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,272 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187071336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,410 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/50be73876534449998df198303e1b935 2024-11-09T21:16:51,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/7f1fc529eae84b9fb5327513f4ae3704 is 50, key is test_row_0/B:col10/1731187010998/Put/seqid=0 2024-11-09T21:16:51,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742132_1308 (size=12301) 2024-11-09T21:16:51,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/7f1fc529eae84b9fb5327513f4ae3704 2024-11-09T21:16:51,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T21:16:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6c476b494382441cb525608bec5637ef is 50, key is test_row_0/C:col10/1731187010998/Put/seqid=0 2024-11-09T21:16:51,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T21:16:51,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742133_1309 (size=12301) 2024-11-09T21:16:51,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187071567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,570 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T21:16:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187071576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:51,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187071647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:51,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:51,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T21:16:51,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:51,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,731 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:51,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T21:16:51,839 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=562 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6c476b494382441cb525608bec5637ef 2024-11-09T21:16:51,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/50be73876534449998df198303e1b935 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/50be73876534449998df198303e1b935 2024-11-09T21:16:51,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/50be73876534449998df198303e1b935, entries=200, sequenceid=562, filesize=14.4 K 2024-11-09T21:16:51,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/7f1fc529eae84b9fb5327513f4ae3704 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/7f1fc529eae84b9fb5327513f4ae3704 2024-11-09T21:16:51,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/7f1fc529eae84b9fb5327513f4ae3704, entries=150, sequenceid=562, filesize=12.0 K 2024-11-09T21:16:51,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/6c476b494382441cb525608bec5637ef as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6c476b494382441cb525608bec5637ef 2024-11-09T21:16:51,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6c476b494382441cb525608bec5637ef, entries=150, sequenceid=562, filesize=12.0 K 2024-11-09T21:16:51,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~436.97 KB/447456, currentSize=26.84 KB/27480 for c7ac4dbed742a563978c6ff5cf7dc21b in 856ms, sequenceid=562, compaction requested=true 2024-11-09T21:16:51,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:51,855 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:51,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:51,856 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:16:51,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:51,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:51,856 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:16:51,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:51,856 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:51,857 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54796 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:16:51,857 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:51,857 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,858 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7ffd9baff3ab471baff77d1166822f23, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f1a748b482904aa296b2e51d81dda854, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/70b9aee494874cbd82c629488ceb572f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/50be73876534449998df198303e1b935] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=53.5 K 2024-11-09T21:16:51,858 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:16:51,858 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:51,858 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,858 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/11e3f5f8e2fe4306b75460ca4b96ff3b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/23311cef913d4f378395866d67fab26c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d583e47449484145926ba5657af4874c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/7f1fc529eae84b9fb5327513f4ae3704] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=46.4 K 2024-11-09T21:16:51,858 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ffd9baff3ab471baff77d1166822f23, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731187005942 2024-11-09T21:16:51,858 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 11e3f5f8e2fe4306b75460ca4b96ff3b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731187005942 2024-11-09T21:16:51,859 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1a748b482904aa296b2e51d81dda854, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731187008183 2024-11-09T21:16:51,859 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 23311cef913d4f378395866d67fab26c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731187008205 2024-11-09T21:16:51,859 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting d583e47449484145926ba5657af4874c, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731187008652 2024-11-09T21:16:51,859 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70b9aee494874cbd82c629488ceb572f, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731187008652 2024-11-09T21:16:51,859 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f1fc529eae84b9fb5327513f4ae3704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731187010938 2024-11-09T21:16:51,860 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50be73876534449998df198303e1b935, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731187010925 2024-11-09T21:16:51,868 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:51,868 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/4a3367a12f2440258ebed54e3b1b43d7 is 50, key is test_row_0/B:col10/1731187010998/Put/seqid=0 2024-11-09T21:16:51,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742134_1310 (size=13153) 2024-11-09T21:16:51,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-09T21:16:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,884 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#265 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:51,884 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-09T21:16:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:51,884 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/64704cd33e664ea48fe15582a5276d07 is 50, key is test_row_0/A:col10/1731187010998/Put/seqid=0 2024-11-09T21:16:51,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:51,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:51,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:51,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:51,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:51,890 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/4a3367a12f2440258ebed54e3b1b43d7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4a3367a12f2440258ebed54e3b1b43d7 2024-11-09T21:16:51,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/83ae40bffa594d51b0addc1d1c456c4a is 50, key is test_row_0/A:col10/1731187011019/Put/seqid=0 2024-11-09T21:16:51,897 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into 4a3367a12f2440258ebed54e3b1b43d7(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:51,897 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:51,897 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=12, startTime=1731187011856; duration=0sec 2024-11-09T21:16:51,897 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:51,897 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:51,897 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:16:51,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742135_1311 (size=13153) 2024-11-09T21:16:51,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47476 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:16:51,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:51,900 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:51,900 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fbd8098fcb0430e95699f28eebb89f1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8110cc189592493d8a8ef7ef80692fae, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/bbedc8600e3c49bc943b8b2c62d13f04, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6c476b494382441cb525608bec5637ef] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=46.4 K 2024-11-09T21:16:51,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fbd8098fcb0430e95699f28eebb89f1, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=303, earliestPutTs=1731187005942 2024-11-09T21:16:51,900 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8110cc189592493d8a8ef7ef80692fae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731187008205 2024-11-09T21:16:51,901 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bbedc8600e3c49bc943b8b2c62d13f04, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=352, earliestPutTs=1731187008652 2024-11-09T21:16:51,902 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c476b494382441cb525608bec5637ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731187010938 2024-11-09T21:16:51,903 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/64704cd33e664ea48fe15582a5276d07 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/64704cd33e664ea48fe15582a5276d07 2024-11-09T21:16:51,908 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 64704cd33e664ea48fe15582a5276d07(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:51,908 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:51,908 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=12, startTime=1731187011855; duration=0sec 2024-11-09T21:16:51,908 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:51,908 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:51,923 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:51,924 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a6eacd6efdc4427983e32635b8ac0366 is 50, key is test_row_0/C:col10/1731187010998/Put/seqid=0 2024-11-09T21:16:51,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742137_1313 (size=13153) 2024-11-09T21:16:51,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742136_1312 (size=9857) 2024-11-09T21:16:51,939 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/a6eacd6efdc4427983e32635b8ac0366 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a6eacd6efdc4427983e32635b8ac0366 2024-11-09T21:16:51,945 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into a6eacd6efdc4427983e32635b8ac0366(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:51,945 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:51,945 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=12, startTime=1731187011856; duration=0sec 2024-11-09T21:16:51,945 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:51,945 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:52,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:52,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187012098, max timestamp=1731187012114Num uniques 859; 2024-11-09T21:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187012098, max timestamp=1731187012114Num uniques 859; 2024-11-09T21:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187012098, max timestamp=1731187012114Num uniques 859; 2024-11-09T21:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:52,117 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:52,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731187012119, max timestamp=1731187012155Num uniques 1068; 2024-11-09T21:16:52,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731187012119, max timestamp=1731187012155Num uniques 1068; 2024-11-09T21:16:52,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-09T21:16:52,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-09T21:16:52,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731187012119, max timestamp=1731187012155Num uniques 1068; 2024-11-09T21:16:52,158 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-09T21:16:52,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731187012179, max timestamp=1731187012185Num uniques 1077; 2024-11-09T21:16:52,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-09T21:16:52,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731187012179, max timestamp=1731187012185Num uniques 1077; 2024-11-09T21:16:52,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-09T21:16:52,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731187012179, max timestamp=1731187012185Num uniques 1077; 2024-11-09T21:16:52,189 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-09T21:16:52,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=286, cellSize=13096, totalHeapSize=29984, min timestamp=1731187012195, max timestamp=1731187012225Num uniques 1086; 2024-11-09T21:16:52,252 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187012211, max timestamp=1731187012225Num uniques 1086; 2024-11-09T21:16:52,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=13096, suffix heap size=140912,new segment heap size=29984 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=286 2024-11-09T21:16:52,253 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-09T21:16:52,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187012211, max timestamp=1731187012225Num uniques 1086; 2024-11-09T21:16:52,255 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-09T21:16:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T21:16:52,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731187012273, max timestamp=1731187012288Num uniques 1145; 2024-11-09T21:16:52,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=52440, new segment data size=11220, suffix heap size=147048,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1145, new segment cells count=245 2024-11-09T21:16:52,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731187012273, max timestamp=1731187012289Num uniques 1095; 2024-11-09T21:16:52,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11210, totalHeapSize=25720, min timestamp=1731187012273, max timestamp=1731187012289Num uniques 1095; 2024-11-09T21:16:52,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=11210, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50140, new segment data size=11210, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187072289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,334 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/83ae40bffa594d51b0addc1d1c456c4a 2024-11-09T21:16:52,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/9dee42f2c87f4b39bda29988857a099b is 50, key is test_row_0/B:col10/1731187011019/Put/seqid=0 2024-11-09T21:16:52,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742138_1314 (size=9857) 2024-11-09T21:16:52,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731187012345, max timestamp=1731187012358Num uniques 1104; 2024-11-09T21:16:52,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-09T21:16:52,361 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731187012345, max timestamp=1731187012358Num uniques 1104; 2024-11-09T21:16:52,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-09T21:16:52,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731187012345, max timestamp=1731187012358Num uniques 1104; 2024-11-09T21:16:52,362 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-09T21:16:52,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731187012374, max timestamp=1731187012399Num uniques 1063; 2024-11-09T21:16:52,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-09T21:16:52,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731187012374, max timestamp=1731187012399Num uniques 1063; 2024-11-09T21:16:52,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731187012374, max timestamp=1731187012399Num uniques 1063; 2024-11-09T21:16:52,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-09T21:16:52,402 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-09T21:16:52,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731187012418, max timestamp=1731187012430Num uniques 1072; 2024-11-09T21:16:52,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-09T21:16:52,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731187012418, max timestamp=1731187012430Num uniques 1072; 2024-11-09T21:16:52,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-09T21:16:52,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731187012418, max timestamp=1731187012430Num uniques 1072; 2024-11-09T21:16:52,434 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=12452, suffix heap size=139456,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=272 2024-11-09T21:16:52,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731187012449, max timestamp=1731187012461Num uniques 1131; 2024-11-09T21:16:52,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-09T21:16:52,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731187012449, max timestamp=1731187012461Num uniques 1131; 2024-11-09T21:16:52,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=281, cellSize=12866, totalHeapSize=29464, min timestamp=1731187012449, max timestamp=1731187012461Num uniques 1131; 2024-11-09T21:16:52,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-09T21:16:52,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51796, new segment data size=12866, suffix heap size=145592,new segment heap size=29464 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1131, new segment cells count=281 2024-11-09T21:16:52,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187072497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187072498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731187012489, max timestamp=1731187012500Num uniques 1140; 2024-11-09T21:16:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-09T21:16:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731187012489, max timestamp=1731187012500Num uniques 1140; 2024-11-09T21:16:52,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-09T21:16:52,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=240, cellSize=10990, totalHeapSize=25200, min timestamp=1731187012489, max timestamp=1731187012500Num uniques 1140; 2024-11-09T21:16:52,504 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=52210, new segment data size=10990, suffix heap size=146528,new segment heap size=25200 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1140, new segment cells count=240 2024-11-09T21:16:52,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731187012532, max timestamp=1731187012581Num uniques 1099; 2024-11-09T21:16:52,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731187012532, max timestamp=1731187012581Num uniques 1099; 2024-11-09T21:16:52,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731187012532, max timestamp=1731187012581Num uniques 1099; 2024-11-09T21:16:52,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-09T21:16:52,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-09T21:16:52,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50334, new segment data size=9114, suffix heap size=142264,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1099, new segment cells count=199 2024-11-09T21:16:52,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731187012611, max timestamp=1731187012629Num uniques 1058; 2024-11-09T21:16:52,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-11-09T21:16:52,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731187012611, max timestamp=1731187012629Num uniques 1058; 2024-11-09T21:16:52,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=258, cellSize=11810, totalHeapSize=27072, min timestamp=1731187012611, max timestamp=1731187012629Num uniques 1058; 2024-11-09T21:16:52,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-11-09T21:16:52,633 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=11810, suffix heap size=138000,new segment heap size=27072 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=258 2024-11-09T21:16:52,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731187012639, max timestamp=1731187012708Num uniques 1117; 2024-11-09T21:16:52,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-09T21:16:52,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=217, cellSize=9932, totalHeapSize=22808, min timestamp=1731187012639, max timestamp=1731187012708Num uniques 1117; 2024-11-09T21:16:52,712 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=9932, suffix heap size=144136,new segment heap size=22808 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=217 2024-11-09T21:16:52,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731187012691, max timestamp=1731187012708Num uniques 1117; 2024-11-09T21:16:52,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=51152, new segment data size=6870, suffix heap size=144136,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1117, new segment cells count=150 2024-11-09T21:16:52,746 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/9dee42f2c87f4b39bda29988857a099b 2024-11-09T21:16:52,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731187012737, max timestamp=1731187012746Num uniques 1076; 2024-11-09T21:16:52,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-09T21:16:52,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731187012737, max timestamp=1731187012746Num uniques 1076; 2024-11-09T21:16:52,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49276, new segment data size=10346, suffix heap size=139872,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1076, new segment cells count=226 2024-11-09T21:16:52,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=226, cellSize=10346, totalHeapSize=23744, min timestamp=1731187012737, max timestamp=1731187012746Num uniques 1009; 2024-11-09T21:16:52,751 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10346, suffix heap size=132904,new segment heap size=23744 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=226 2024-11-09T21:16:52,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/9bc84794d40641e2bef56a3515646eb1 is 50, key is test_row_0/C:col10/1731187011019/Put/seqid=0 2024-11-09T21:16:52,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742139_1315 (size=9857) 2024-11-09T21:16:52,761 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/9bc84794d40641e2bef56a3515646eb1 2024-11-09T21:16:52,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/83ae40bffa594d51b0addc1d1c456c4a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/83ae40bffa594d51b0addc1d1c456c4a 2024-11-09T21:16:52,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/83ae40bffa594d51b0addc1d1c456c4a, entries=100, sequenceid=569, filesize=9.6 K 2024-11-09T21:16:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/9dee42f2c87f4b39bda29988857a099b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9dee42f2c87f4b39bda29988857a099b 2024-11-09T21:16:52,776 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9dee42f2c87f4b39bda29988857a099b, entries=100, sequenceid=569, filesize=9.6 K 2024-11-09T21:16:52,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/9bc84794d40641e2bef56a3515646eb1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9bc84794d40641e2bef56a3515646eb1 2024-11-09T21:16:52,782 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9bc84794d40641e2bef56a3515646eb1, entries=100, sequenceid=569, filesize=9.6 K 2024-11-09T21:16:52,783 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=120.76 KB/123660 for c7ac4dbed742a563978c6ff5cf7dc21b in 899ms, sequenceid=569, compaction requested=false 2024-11-09T21:16:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:52,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-09T21:16:52,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-09T21:16:52,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-09T21:16:52,787 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6650 sec 2024-11-09T21:16:52,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:52,787 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=127.47 KB heapSize=358.78 KB 2024-11-09T21:16:52,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:52,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:52,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:52,787 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:52,788 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:52,790 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.6710 sec 2024-11-09T21:16:52,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/5b736a228f164dd8b554bc1c8c85bb6a is 50, key is test_row_0/A:col10/1731187012785/Put/seqid=0 2024-11-09T21:16:52,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 228 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187072801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 263 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187072803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742140_1316 (size=17181) 2024-11-09T21:16:52,810 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/5b736a228f164dd8b554bc1c8c85bb6a 2024-11-09T21:16:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187072804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/be588b32c8fe4831a0d19b3d6ff3ef47 is 50, key is test_row_0/B:col10/1731187012785/Put/seqid=0 2024-11-09T21:16:52,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742141_1317 (size=12301) 2024-11-09T21:16:52,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187072836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,839 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,839 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187072846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,851 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,852 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187072916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 229 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187072916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:52,919 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 264 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187072916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:52,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 239 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187073127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 230 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187073127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 265 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187073127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:53,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/be588b32c8fe4831a0d19b3d6ff3ef47 2024-11-09T21:16:53,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5677ce726d56423293a0a5fa7d9e41f5 is 50, key is test_row_0/C:col10/1731187012785/Put/seqid=0 2024-11-09T21:16:53,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742142_1318 (size=12301) 2024-11-09T21:16:53,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-09T21:16:53,266 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:53,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:53,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-09T21:16:53,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T21:16:53,268 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:53,269 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:53,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:53,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T21:16:53,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T21:16:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:53,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:53,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:53,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 240 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187073436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,441 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:53,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 266 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187073437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:53,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 231 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187073437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:53,442 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:53,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T21:16:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:53,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:53,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:53,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T21:16:53,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=831 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5677ce726d56423293a0a5fa7d9e41f5 2024-11-09T21:16:53,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/5b736a228f164dd8b554bc1c8c85bb6a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5b736a228f164dd8b554bc1c8c85bb6a 2024-11-09T21:16:53,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5b736a228f164dd8b554bc1c8c85bb6a, entries=250, sequenceid=831, filesize=16.8 K 2024-11-09T21:16:53,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/be588b32c8fe4831a0d19b3d6ff3ef47 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/be588b32c8fe4831a0d19b3d6ff3ef47 2024-11-09T21:16:53,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/be588b32c8fe4831a0d19b3d6ff3ef47, entries=150, sequenceid=831, filesize=12.0 K 2024-11-09T21:16:53,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/5677ce726d56423293a0a5fa7d9e41f5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5677ce726d56423293a0a5fa7d9e41f5 2024-11-09T21:16:53,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5677ce726d56423293a0a5fa7d9e41f5, entries=150, sequenceid=831, filesize=12.0 K 2024-11-09T21:16:53,658 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~378.66 KB/387744, currentSize=46.96 KB/48090 for c7ac4dbed742a563978c6ff5cf7dc21b in 871ms, sequenceid=831, compaction requested=true 2024-11-09T21:16:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:53,658 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:53,659 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:53,659 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:53,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:53,659 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:53,659 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40191 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:53,659 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:53,659 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:53,660 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,660 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,660 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4a3367a12f2440258ebed54e3b1b43d7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9dee42f2c87f4b39bda29988857a099b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/be588b32c8fe4831a0d19b3d6ff3ef47] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=34.5 K 2024-11-09T21:16:53,660 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/64704cd33e664ea48fe15582a5276d07, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/83ae40bffa594d51b0addc1d1c456c4a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5b736a228f164dd8b554bc1c8c85bb6a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=39.2 K 2024-11-09T21:16:53,660 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4a3367a12f2440258ebed54e3b1b43d7, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731187010938 2024-11-09T21:16:53,660 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64704cd33e664ea48fe15582a5276d07, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731187010938 2024-11-09T21:16:53,660 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dee42f2c87f4b39bda29988857a099b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731187011014 2024-11-09T21:16:53,661 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83ae40bffa594d51b0addc1d1c456c4a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731187011014 2024-11-09T21:16:53,661 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting be588b32c8fe4831a0d19b3d6ff3ef47, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1731187012779 2024-11-09T21:16:53,661 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b736a228f164dd8b554bc1c8c85bb6a, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1731187012765 2024-11-09T21:16:53,668 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:53,669 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b3482dabb5f8455bb2fa6618d789e7e7 is 50, key is test_row_0/A:col10/1731187012785/Put/seqid=0 2024-11-09T21:16:53,674 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#274 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:53,675 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/418affb6d64a49af921772c8d98666b8 is 50, key is test_row_0/B:col10/1731187012785/Put/seqid=0 2024-11-09T21:16:53,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742143_1319 (size=13255) 2024-11-09T21:16:53,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742144_1320 (size=13255) 2024-11-09T21:16:53,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:53,727 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:53,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:53,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/bbfa39e5b8e04234adef33082cffde6e is 50, key is test_row_0/A:col10/1731187012801/Put/seqid=0 2024-11-09T21:16:53,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742145_1321 (size=12301) 2024-11-09T21:16:53,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T21:16:53,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:53,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187013975, max timestamp=1731187013981Num uniques 859; 2024-11-09T21:16:53,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:53,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187013975, max timestamp=1731187013981Num uniques 859; 2024-11-09T21:16:53,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:53,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187013975, max timestamp=1731187013981Num uniques 859; 2024-11-09T21:16:53,984 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:54,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187074011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 253 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187074012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,017 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187074012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,018 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,085 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/b3482dabb5f8455bb2fa6618d789e7e7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b3482dabb5f8455bb2fa6618d789e7e7 2024-11-09T21:16:54,101 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/418affb6d64a49af921772c8d98666b8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/418affb6d64a49af921772c8d98666b8 2024-11-09T21:16:54,102 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into b3482dabb5f8455bb2fa6618d789e7e7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:54,102 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:54,102 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187013658; duration=0sec 2024-11-09T21:16:54,102 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:54,102 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:54,102 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:54,105 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35311 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:54,105 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:54,105 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,105 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a6eacd6efdc4427983e32635b8ac0366, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9bc84794d40641e2bef56a3515646eb1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5677ce726d56423293a0a5fa7d9e41f5] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=34.5 K 2024-11-09T21:16:54,106 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6eacd6efdc4427983e32635b8ac0366, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=562, earliestPutTs=1731187010938 2024-11-09T21:16:54,106 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bc84794d40641e2bef56a3515646eb1, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=569, earliestPutTs=1731187011014 2024-11-09T21:16:54,107 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5677ce726d56423293a0a5fa7d9e41f5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1731187012779 2024-11-09T21:16:54,109 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into 418affb6d64a49af921772c8d98666b8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:54,109 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:54,109 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187013659; duration=0sec 2024-11-09T21:16:54,109 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:54,109 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:54,122 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#276 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:54,123 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/ccfe23e378e24af6b0a91b8575a589da is 50, key is test_row_0/C:col10/1731187012785/Put/seqid=0 2024-11-09T21:16:54,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187074127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187074127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,128 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187074127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,129 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742146_1322 (size=13255) 2024-11-09T21:16:54,137 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/ccfe23e378e24af6b0a91b8575a589da as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ccfe23e378e24af6b0a91b8575a589da 2024-11-09T21:16:54,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=841 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/bbfa39e5b8e04234adef33082cffde6e 2024-11-09T21:16:54,145 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into ccfe23e378e24af6b0a91b8575a589da(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:54,146 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:54,146 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187013659; duration=0sec 2024-11-09T21:16:54,146 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:54,146 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:54,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/72586780ab114d3c880274c77ec020e9 is 50, key is test_row_0/B:col10/1731187012801/Put/seqid=0 2024-11-09T21:16:54,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742147_1323 (size=12301) 2024-11-09T21:16:54,165 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=841 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/72586780ab114d3c880274c77ec020e9 2024-11-09T21:16:54,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/833debfc82104d7b82829b4078d0021f is 50, key is test_row_0/C:col10/1731187012801/Put/seqid=0 2024-11-09T21:16:54,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742148_1324 (size=12301) 2024-11-09T21:16:54,176 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=841 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/833debfc82104d7b82829b4078d0021f 2024-11-09T21:16:54,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/bbfa39e5b8e04234adef33082cffde6e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/bbfa39e5b8e04234adef33082cffde6e 2024-11-09T21:16:54,187 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/bbfa39e5b8e04234adef33082cffde6e, entries=150, sequenceid=841, filesize=12.0 K 2024-11-09T21:16:54,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/72586780ab114d3c880274c77ec020e9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/72586780ab114d3c880274c77ec020e9 2024-11-09T21:16:54,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/72586780ab114d3c880274c77ec020e9, entries=150, sequenceid=841, filesize=12.0 K 2024-11-09T21:16:54,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/833debfc82104d7b82829b4078d0021f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/833debfc82104d7b82829b4078d0021f 2024-11-09T21:16:54,197 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/833debfc82104d7b82829b4078d0021f, entries=150, sequenceid=841, filesize=12.0 K 2024-11-09T21:16:54,198 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for c7ac4dbed742a563978c6ff5cf7dc21b in 470ms, sequenceid=841, compaction requested=false 2024-11-09T21:16:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-09T21:16:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-09T21:16:54,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-09T21:16:54,201 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 930 msec 2024-11-09T21:16:54,202 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 934 msec 2024-11-09T21:16:54,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:54,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187013997, max timestamp=1731187014338Num uniques 1068; 2024-11-09T21:16:54,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T21:16:54,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187013997, max timestamp=1731187014338Num uniques 1068; 2024-11-09T21:16:54,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T21:16:54,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187013997, max timestamp=1731187014338Num uniques 1068; 2024-11-09T21:16:54,342 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T21:16:54,343 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=67.09 KB heapSize=183.88 KB 2024-11-09T21:16:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:54,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:54,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/7a1fed30fc9c4911940b7e18292864e4 is 50, key is test_row_0/A:col10/1731187014005/Put/seqid=0 2024-11-09T21:16:54,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742149_1325 (size=12301) 2024-11-09T21:16:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187014379, max timestamp=1731187014398Num uniques 859; 2024-11-09T21:16:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187014379, max timestamp=1731187014398Num uniques 859; 2024-11-09T21:16:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187014379, max timestamp=1731187014398Num uniques 859; 2024-11-09T21:16:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:54,401 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:16:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-09T21:16:54,406 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:54,407 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-09T21:16:54,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T21:16:54,408 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:54,409 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:54,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:54,443 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187014427, max timestamp=1731187014440Num uniques 1068; 2024-11-09T21:16:54,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 258 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187074435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T21:16:54,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187014427, max timestamp=1731187014440Num uniques 1068; 2024-11-09T21:16:54,443 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T21:16:54,443 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731187014427, max timestamp=1731187014440Num uniques 1068; 2024-11-09T21:16:54,444 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-09T21:16:54,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731187014493, max timestamp=1731187014506Num uniques 1027; 2024-11-09T21:16:54,509 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T21:16:54,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731187014493, max timestamp=1731187014506Num uniques 1027; 2024-11-09T21:16:54,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731187014493, max timestamp=1731187014506Num uniques 1027; 2024-11-09T21:16:54,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T21:16:54,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-09T21:16:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T21:16:54,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187014529, max timestamp=1731187014549Num uniques 1036; 2024-11-09T21:16:54,553 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T21:16:54,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187014529, max timestamp=1731187014549Num uniques 1036; 2024-11-09T21:16:54,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T21:16:54,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731187014529, max timestamp=1731187014549Num uniques 1036; 2024-11-09T21:16:54,554 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=10806, suffix heap size=135712,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=236 2024-11-09T21:16:54,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 315 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187074580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731187014563, max timestamp=1731187014582Num uniques 1095; 2024-11-09T21:16:54,585 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,585 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731187014563, max timestamp=1731187014582Num uniques 1095; 2024-11-09T21:16:54,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:54,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731187014563, max timestamp=1731187014582Num uniques 1095; 2024-11-09T21:16:54,586 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-09T21:16:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 274 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187074627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 298 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187074627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,632 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187074697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T21:16:54,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 275 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187074736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 299 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187074736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,742 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=884 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/7a1fed30fc9c4911940b7e18292864e4 2024-11-09T21:16:54,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/5a470733833141ae98f9894f696d9b94 is 50, key is test_row_0/B:col10/1731187014005/Put/seqid=0 2024-11-09T21:16:54,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742150_1326 (size=12301) 2024-11-09T21:16:54,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:54,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:54,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187074906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,910 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 276 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187074946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 300 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187074946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:54,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,021 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:55,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T21:16:55,174 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:55,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:55,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=884 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/5a470733833141ae98f9894f696d9b94 2024-11-09T21:16:55,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/50ef30eaa4864904b87cb41117364700 is 50, key is test_row_0/C:col10/1731187014005/Put/seqid=0 2024-11-09T21:16:55,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742151_1327 (size=12301) 2024-11-09T21:16:55,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187075217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 277 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187075256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 301 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187075256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,260 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:55,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,327 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,479 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:55,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T21:16:55,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=884 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/50ef30eaa4864904b87cb41117364700 2024-11-09T21:16:55,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/7a1fed30fc9c4911940b7e18292864e4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7a1fed30fc9c4911940b7e18292864e4 2024-11-09T21:16:55,604 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7a1fed30fc9c4911940b7e18292864e4, entries=150, sequenceid=884, filesize=12.0 K 2024-11-09T21:16:55,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/5a470733833141ae98f9894f696d9b94 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/5a470733833141ae98f9894f696d9b94 2024-11-09T21:16:55,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/5a470733833141ae98f9894f696d9b94, entries=150, sequenceid=884, filesize=12.0 K 2024-11-09T21:16:55,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/50ef30eaa4864904b87cb41117364700 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/50ef30eaa4864904b87cb41117364700 2024-11-09T21:16:55,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/50ef30eaa4864904b87cb41117364700, entries=150, sequenceid=884, filesize=12.0 K 2024-11-09T21:16:55,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~105.19 KB/107712, currentSize=147.60 KB/151140 for c7ac4dbed742a563978c6ff5cf7dc21b in 1270ms, sequenceid=884, compaction requested=true 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:55,612 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:55,612 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:55,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:55,613 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:55,613 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:55,613 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:55,613 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:55,613 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,613 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,613 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/418affb6d64a49af921772c8d98666b8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/72586780ab114d3c880274c77ec020e9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/5a470733833141ae98f9894f696d9b94] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.0 K 2024-11-09T21:16:55,613 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b3482dabb5f8455bb2fa6618d789e7e7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/bbfa39e5b8e04234adef33082cffde6e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7a1fed30fc9c4911940b7e18292864e4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.0 K 2024-11-09T21:16:55,613 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 418affb6d64a49af921772c8d98666b8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1731187012779 2024-11-09T21:16:55,613 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3482dabb5f8455bb2fa6618d789e7e7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1731187012779 2024-11-09T21:16:55,614 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbfa39e5b8e04234adef33082cffde6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=841, earliestPutTs=1731187012795 2024-11-09T21:16:55,614 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 72586780ab114d3c880274c77ec020e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=841, earliestPutTs=1731187012795 2024-11-09T21:16:55,614 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a1fed30fc9c4911940b7e18292864e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=884, earliestPutTs=1731187014005 2024-11-09T21:16:55,614 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a470733833141ae98f9894f696d9b94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=884, earliestPutTs=1731187014005 2024-11-09T21:16:55,622 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:55,622 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#283 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:55,622 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/48a879b14c5542f69257a4b3dd2ca102 is 50, key is test_row_0/A:col10/1731187014005/Put/seqid=0 2024-11-09T21:16:55,624 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b44a23a6825c438891dff59ba29dacb8 is 50, key is test_row_0/B:col10/1731187014005/Put/seqid=0 2024-11-09T21:16:55,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,632 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=147.60 KB heapSize=416.77 KB 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:55,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:55,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742153_1329 (size=13357) 2024-11-09T21:16:55,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742152_1328 (size=13357) 2024-11-09T21:16:55,645 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/48a879b14c5542f69257a4b3dd2ca102 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/48a879b14c5542f69257a4b3dd2ca102 2024-11-09T21:16:55,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/e25e980c745249d395a93ed9ada68600 is 50, key is test_row_0/A:col10/1731187014625/Put/seqid=0 2024-11-09T21:16:55,652 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 48a879b14c5542f69257a4b3dd2ca102(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:55,652 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:55,652 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187015612; duration=0sec 2024-11-09T21:16:55,652 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:55,652 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:55,652 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:55,654 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:55,654 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:55,655 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:55,655 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ccfe23e378e24af6b0a91b8575a589da, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/833debfc82104d7b82829b4078d0021f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/50ef30eaa4864904b87cb41117364700] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.0 K 2024-11-09T21:16:55,655 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccfe23e378e24af6b0a91b8575a589da, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=831, earliestPutTs=1731187012779 2024-11-09T21:16:55,655 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 833debfc82104d7b82829b4078d0021f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=841, earliestPutTs=1731187012795 2024-11-09T21:16:55,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50ef30eaa4864904b87cb41117364700, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=884, earliestPutTs=1731187014005 2024-11-09T21:16:55,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742154_1330 (size=12301) 2024-11-09T21:16:55,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=990 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/e25e980c745249d395a93ed9ada68600 2024-11-09T21:16:55,663 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#285 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:55,663 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/4ac2ca3472c34a2d910f0df3c2fa6f56 is 50, key is test_row_0/C:col10/1731187014005/Put/seqid=0 2024-11-09T21:16:55,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/1387a7d7ea864aae96b50d8f19071448 is 50, key is test_row_0/B:col10/1731187014625/Put/seqid=0 2024-11-09T21:16:55,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742155_1331 (size=13357) 2024-11-09T21:16:55,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742156_1332 (size=12301) 2024-11-09T21:16:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:55,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:55,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187075751, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,754 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 278 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187075766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 302 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187075766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:55,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:55,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187075856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:55,859 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,045 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/b44a23a6825c438891dff59ba29dacb8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b44a23a6825c438891dff59ba29dacb8 2024-11-09T21:16:56,049 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into b44a23a6825c438891dff59ba29dacb8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:56,049 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:56,049 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187015612; duration=0sec 2024-11-09T21:16:56,049 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:56,049 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187076066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,070 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/4ac2ca3472c34a2d910f0df3c2fa6f56 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4ac2ca3472c34a2d910f0df3c2fa6f56 2024-11-09T21:16:56,074 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 4ac2ca3472c34a2d910f0df3c2fa6f56(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:56,074 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:56,074 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187015612; duration=0sec 2024-11-09T21:16:56,074 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:56,074 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:56,076 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=990 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/1387a7d7ea864aae96b50d8f19071448 2024-11-09T21:16:56,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/c22120763dd14a4ab3528c6a042d2a3b is 50, key is test_row_0/C:col10/1731187014625/Put/seqid=0 2024-11-09T21:16:56,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742157_1333 (size=12301) 2024-11-09T21:16:56,094 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=990 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/c22120763dd14a4ab3528c6a042d2a3b 2024-11-09T21:16:56,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/e25e980c745249d395a93ed9ada68600 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e25e980c745249d395a93ed9ada68600 2024-11-09T21:16:56,103 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e25e980c745249d395a93ed9ada68600, entries=150, sequenceid=990, filesize=12.0 K 2024-11-09T21:16:56,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/1387a7d7ea864aae96b50d8f19071448 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1387a7d7ea864aae96b50d8f19071448 2024-11-09T21:16:56,108 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1387a7d7ea864aae96b50d8f19071448, entries=150, sequenceid=990, filesize=12.0 K 2024-11-09T21:16:56,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/c22120763dd14a4ab3528c6a042d2a3b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/c22120763dd14a4ab3528c6a042d2a3b 2024-11-09T21:16:56,113 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/c22120763dd14a4ab3528c6a042d2a3b, entries=150, sequenceid=990, filesize=12.0 K 2024-11-09T21:16:56,114 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~416.72 KB/426720, currentSize=33.54 KB/34350 for c7ac4dbed742a563978c6ff5cf7dc21b in 482ms, sequenceid=990, compaction requested=false 2024-11-09T21:16:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:56,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-09T21:16:56,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-09T21:16:56,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-09T21:16:56,117 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7070 sec 2024-11-09T21:16:56,119 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 1.7110 sec 2024-11-09T21:16:56,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:56,381 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:56,382 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:16:56,385 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3d9420f2ff0a4fe381882438c4b6c880 is 50, key is test_row_0/A:col10/1731187016380/Put/seqid=0 2024-11-09T21:16:56,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742158_1334 (size=14741) 2024-11-09T21:16:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187016458, max timestamp=1731187016476Num uniques 859; 2024-11-09T21:16:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187016458, max timestamp=1731187016476Num uniques 859; 2024-11-09T21:16:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187016458, max timestamp=1731187016476Num uniques 859; 2024-11-09T21:16:56,479 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:16:56,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-09T21:16:56,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:56,557 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:16:56,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-09T21:16:56,558 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:16:56,559 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:16:56,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:16:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T21:16:56,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187076573, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T21:16:56,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187076686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,690 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T21:16:56,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:56,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:56,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:56,712 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:56,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:56,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55052 deadline: 1731187076776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 303 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55056 deadline: 1731187076776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,779 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:56,779 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2152 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,780 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1003 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3d9420f2ff0a4fe381882438c4b6c880 2024-11-09T21:16:56,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/8aa1b86dd10f472ba38da534059ed179 is 50, key is test_row_0/B:col10/1731187016380/Put/seqid=0 2024-11-09T21:16:56,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742159_1335 (size=12301) 2024-11-09T21:16:56,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1003 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/8aa1b86dd10f472ba38da534059ed179 2024-11-09T21:16:56,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/dc31e68c78a64ba680f88339cce61090 is 50, key is test_row_0/C:col10/1731187016380/Put/seqid=0 2024-11-09T21:16:56,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742160_1336 (size=12301) 2024-11-09T21:16:56,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T21:16:56,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:56,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:56,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:56,865 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:56,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:56,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T21:16:56,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55104 deadline: 1731187076876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,879 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8225 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187076896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55066 deadline: 1731187076896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:56,898 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in c7ac4dbed742a563978c6ff5cf7dc21b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:16:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:56,899 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:57,018 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T21:16:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,018 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:57,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:57,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:57,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T21:16:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:57,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:57,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:16:57,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T21:16:57,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:16:57,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:55094 deadline: 1731187077206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:16:57,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=1003 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/dc31e68c78a64ba680f88339cce61090 2024-11-09T21:16:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:16:57,208 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:16:57,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/3d9420f2ff0a4fe381882438c4b6c880 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3d9420f2ff0a4fe381882438c4b6c880 2024-11-09T21:16:57,214 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3d9420f2ff0a4fe381882438c4b6c880, entries=200, sequenceid=1003, filesize=14.4 K 2024-11-09T21:16:57,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/8aa1b86dd10f472ba38da534059ed179 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/8aa1b86dd10f472ba38da534059ed179 2024-11-09T21:16:57,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/8aa1b86dd10f472ba38da534059ed179, entries=150, sequenceid=1003, filesize=12.0 K 2024-11-09T21:16:57,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/dc31e68c78a64ba680f88339cce61090 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/dc31e68c78a64ba680f88339cce61090 2024-11-09T21:16:57,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/dc31e68c78a64ba680f88339cce61090, entries=150, sequenceid=1003, filesize=12.0 K 2024-11-09T21:16:57,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for c7ac4dbed742a563978c6ff5cf7dc21b in 842ms, sequenceid=1003, compaction requested=true 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:57,223 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:16:57,223 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store c7ac4dbed742a563978c6ff5cf7dc21b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:16:57,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:57,224 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:57,224 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:57,225 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/A is initiating minor compaction (all files) 2024-11-09T21:16:57,225 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/B is initiating minor compaction (all files) 2024-11-09T21:16:57,225 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/A in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,225 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/B in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,225 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/48a879b14c5542f69257a4b3dd2ca102, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e25e980c745249d395a93ed9ada68600, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3d9420f2ff0a4fe381882438c4b6c880] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=39.5 K 2024-11-09T21:16:57,225 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b44a23a6825c438891dff59ba29dacb8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1387a7d7ea864aae96b50d8f19071448, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/8aa1b86dd10f472ba38da534059ed179] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.1 K 2024-11-09T21:16:57,225 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b44a23a6825c438891dff59ba29dacb8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=884, earliestPutTs=1731187014005 2024-11-09T21:16:57,225 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48a879b14c5542f69257a4b3dd2ca102, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=884, earliestPutTs=1731187014005 2024-11-09T21:16:57,225 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1387a7d7ea864aae96b50d8f19071448, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1731187014613 2024-11-09T21:16:57,225 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25e980c745249d395a93ed9ada68600, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1731187014613 2024-11-09T21:16:57,226 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa1b86dd10f472ba38da534059ed179, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1003, earliestPutTs=1731187015747 2024-11-09T21:16:57,226 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d9420f2ff0a4fe381882438c4b6c880, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=1003, earliestPutTs=1731187015741 2024-11-09T21:16:57,232 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#B#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:57,233 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/6bfa2c9eb0ad4e8db123a4c56303bc9d is 50, key is test_row_0/B:col10/1731187016380/Put/seqid=0 2024-11-09T21:16:57,238 INFO [Thread-1108 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1108. 2024-11-09T21:16:57,238 DEBUG [Thread-1108 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:57,238 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,238 DEBUG [Thread-1108 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:57,240 INFO [Thread-1106 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1106. 2024-11-09T21:16:57,240 DEBUG [Thread-1106 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:57,240 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,240 DEBUG [Thread-1106 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,240 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:57,241 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#A#compaction#292 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:57,242 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/9255b68fbb5c43cd851de16187bbe702 is 50, key is test_row_0/A:col10/1731187016380/Put/seqid=0 2024-11-09T21:16:57,242 INFO [Thread-1112 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1112. 2024-11-09T21:16:57,242 DEBUG [Thread-1112 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:57,242 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,243 DEBUG [Thread-1112 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:57,243 INFO [Thread-1114 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1114. 2024-11-09T21:16:57,243 DEBUG [Thread-1114 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:57,243 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,243 DEBUG [Thread-1114 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,243 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:57,244 INFO [Thread-1110 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1110. 2024-11-09T21:16:57,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742161_1337 (size=13459) 2024-11-09T21:16:57,244 DEBUG [Thread-1110 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:57,244 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,244 DEBUG [Thread-1110 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,244 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:57,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742162_1338 (size=13459) 2024-11-09T21:16:57,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-09T21:16:57,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-09T21:16:57,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:16:57,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:57,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:16:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:16:57,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:16:57,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/c754ed41cda344da9da3b6399a9fceba is 50, key is test_row_0/A:col10/1731187016572/Put/seqid=0 2024-11-09T21:16:57,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742163_1339 (size=12301) 2024-11-09T21:16:57,656 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/6bfa2c9eb0ad4e8db123a4c56303bc9d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/6bfa2c9eb0ad4e8db123a4c56303bc9d 2024-11-09T21:16:57,657 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/9255b68fbb5c43cd851de16187bbe702 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9255b68fbb5c43cd851de16187bbe702 2024-11-09T21:16:57,660 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/B of c7ac4dbed742a563978c6ff5cf7dc21b into 6bfa2c9eb0ad4e8db123a4c56303bc9d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:57,660 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:57,660 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/B, priority=13, startTime=1731187017223; duration=0sec 2024-11-09T21:16:57,660 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:16:57,660 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:B 2024-11-09T21:16:57,661 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:16:57,661 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/A of c7ac4dbed742a563978c6ff5cf7dc21b into 9255b68fbb5c43cd851de16187bbe702(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:57,661 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:57,661 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/A, priority=13, startTime=1731187017223; duration=0sec 2024-11-09T21:16:57,661 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:57,661 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:A 2024-11-09T21:16:57,661 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:16:57,661 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): c7ac4dbed742a563978c6ff5cf7dc21b/C is initiating minor compaction (all files) 2024-11-09T21:16:57,661 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of c7ac4dbed742a563978c6ff5cf7dc21b/C in TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:57,661 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4ac2ca3472c34a2d910f0df3c2fa6f56, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/c22120763dd14a4ab3528c6a042d2a3b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/dc31e68c78a64ba680f88339cce61090] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp, totalSize=37.1 K 2024-11-09T21:16:57,662 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ac2ca3472c34a2d910f0df3c2fa6f56, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=884, earliestPutTs=1731187014005 2024-11-09T21:16:57,662 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c22120763dd14a4ab3528c6a042d2a3b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=990, earliestPutTs=1731187014613 2024-11-09T21:16:57,662 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting dc31e68c78a64ba680f88339cce61090, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=1003, earliestPutTs=1731187015747 2024-11-09T21:16:57,667 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): c7ac4dbed742a563978c6ff5cf7dc21b#C#compaction#294 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:16:57,668 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/4bb8668e52b04302991482b331de64a2 is 50, key is test_row_0/C:col10/1731187016380/Put/seqid=0 2024-11-09T21:16:57,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742164_1340 (size=13459) 2024-11-09T21:16:57,675 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/4bb8668e52b04302991482b331de64a2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4bb8668e52b04302991482b331de64a2 2024-11-09T21:16:57,678 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in c7ac4dbed742a563978c6ff5cf7dc21b/C of c7ac4dbed742a563978c6ff5cf7dc21b into 4bb8668e52b04302991482b331de64a2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:16:57,678 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:57,678 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., storeName=c7ac4dbed742a563978c6ff5cf7dc21b/C, priority=13, startTime=1731187017223; duration=0sec 2024-11-09T21:16:57,679 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:16:57,679 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: c7ac4dbed742a563978c6ff5cf7dc21b:C 2024-11-09T21:16:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T21:16:57,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:16:57,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. as already flushing 2024-11-09T21:16:57,718 INFO [Thread-1097 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1097. 2024-11-09T21:16:57,718 DEBUG [Thread-1097 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:57,718 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,718 DEBUG [Thread-1097 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:57,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:57,740 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1040 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/c754ed41cda344da9da3b6399a9fceba 2024-11-09T21:16:57,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/da40fd3a781a4f2dafab5153012a6659 is 50, key is test_row_0/B:col10/1731187016572/Put/seqid=0 2024-11-09T21:16:57,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742165_1341 (size=12301) 2024-11-09T21:16:58,151 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1040 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/da40fd3a781a4f2dafab5153012a6659 2024-11-09T21:16:58,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/26eb34684d4d4af2b0d8b7a3d1662450 is 50, key is test_row_0/C:col10/1731187016572/Put/seqid=0 2024-11-09T21:16:58,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742166_1342 (size=12301) 2024-11-09T21:16:58,566 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=1040 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/26eb34684d4d4af2b0d8b7a3d1662450 2024-11-09T21:16:58,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/c754ed41cda344da9da3b6399a9fceba as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/c754ed41cda344da9da3b6399a9fceba 2024-11-09T21:16:58,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/c754ed41cda344da9da3b6399a9fceba, entries=150, sequenceid=1040, filesize=12.0 K 2024-11-09T21:16:58,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/da40fd3a781a4f2dafab5153012a6659 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/da40fd3a781a4f2dafab5153012a6659 2024-11-09T21:16:58,585 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/da40fd3a781a4f2dafab5153012a6659, entries=150, sequenceid=1040, filesize=12.0 K 2024-11-09T21:16:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/26eb34684d4d4af2b0d8b7a3d1662450 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/26eb34684d4d4af2b0d8b7a3d1662450 2024-11-09T21:16:58,589 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/26eb34684d4d4af2b0d8b7a3d1662450, entries=150, sequenceid=1040, filesize=12.0 K 2024-11-09T21:16:58,589 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=6.71 KB/6870 for c7ac4dbed742a563978c6ff5cf7dc21b in 1264ms, sequenceid=1040, compaction requested=false 2024-11-09T21:16:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for c7ac4dbed742a563978c6ff5cf7dc21b: 2024-11-09T21:16:58,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:16:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-09T21:16:58,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-09T21:16:58,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-09T21:16:58,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0320 sec 2024-11-09T21:16:58,594 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.0360 sec 2024-11-09T21:16:58,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-09T21:16:58,707 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:16:58,799 INFO [Thread-1099 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1099. 2024-11-09T21:16:58,799 DEBUG [Thread-1099 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:58,799 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:58,799 DEBUG [Thread-1099 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:58,799 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:16:58,808 INFO [Thread-1095 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1095. 2024-11-09T21:16:58,808 DEBUG [Thread-1095 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:16:58,808 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:58,808 DEBUG [Thread-1095 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:16:58,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:03,223 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:17:06,961 INFO [Thread-1101 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1101. 2024-11-09T21:17:06,961 DEBUG [Thread-1101 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:06,961 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:06,962 DEBUG [Thread-1101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:06,962 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:06,990 INFO [Thread-1103 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1103. 2024-11-09T21:17:06,990 DEBUG [Thread-1103 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:06,990 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:06,991 DEBUG [Thread-1103 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:06,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:06,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T21:17:06,991 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 247 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 326 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 241 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 63 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2828 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2920 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2990 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2822 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2930 2024-11-09T21:17:06,992 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T21:17:06,992 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:17:06,993 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:17:06,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:06,993 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:06,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:06,995 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T21:17:06,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:06,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T21:17:06,999 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187026999"}]},"ts":"1731187026999"} 2024-11-09T21:17:07,001 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T21:17:07,001 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T21:17:07,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:17:07,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, UNASSIGN}] 2024-11-09T21:17:07,005 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, UNASSIGN 2024-11-09T21:17:07,006 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c7ac4dbed742a563978c6ff5cf7dc21b, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:07,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, UNASSIGN because future has completed 2024-11-09T21:17:07,008 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:17:07,008 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:17:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T21:17:07,162 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:17:07,162 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:17:07,162 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing c7ac4dbed742a563978c6ff5cf7dc21b, disabling compactions & flushes 2024-11-09T21:17:07,163 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:17:07,163 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:17:07,163 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. after waiting 0 ms 2024-11-09T21:17:07,163 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:17:07,163 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing c7ac4dbed742a563978c6ff5cf7dc21b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T21:17:07,164 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=A 2024-11-09T21:17:07,164 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:07,164 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=B 2024-11-09T21:17:07,164 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:07,164 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactingMemStore(205): FLUSHING TO DISK c7ac4dbed742a563978c6ff5cf7dc21b, store=C 2024-11-09T21:17:07,164 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:07,171 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/e61f9e10b49a4cd49b51cf01e9f434be is 50, key is test_row_0/A:col10/1731187026987/Put/seqid=0 2024-11-09T21:17:07,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742167_1343 (size=12301) 2024-11-09T21:17:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T21:17:07,576 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1051 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/e61f9e10b49a4cd49b51cf01e9f434be 2024-11-09T21:17:07,591 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/4340055ac0be4789a48d08368443130f is 50, key is test_row_0/B:col10/1731187026987/Put/seqid=0 2024-11-09T21:17:07,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742168_1344 (size=12301) 2024-11-09T21:17:07,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T21:17:07,995 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1051 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/4340055ac0be4789a48d08368443130f 2024-11-09T21:17:08,003 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/54af00c9f7384f5c886d4fe20b201760 is 50, key is test_row_0/C:col10/1731187026987/Put/seqid=0 2024-11-09T21:17:08,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742169_1345 (size=12301) 2024-11-09T21:17:08,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T21:17:08,407 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=1051 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/54af00c9f7384f5c886d4fe20b201760 2024-11-09T21:17:08,411 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/A/e61f9e10b49a4cd49b51cf01e9f434be as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e61f9e10b49a4cd49b51cf01e9f434be 2024-11-09T21:17:08,415 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e61f9e10b49a4cd49b51cf01e9f434be, entries=150, sequenceid=1051, filesize=12.0 K 2024-11-09T21:17:08,416 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/B/4340055ac0be4789a48d08368443130f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4340055ac0be4789a48d08368443130f 2024-11-09T21:17:08,420 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4340055ac0be4789a48d08368443130f, entries=150, sequenceid=1051, filesize=12.0 K 2024-11-09T21:17:08,421 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/.tmp/C/54af00c9f7384f5c886d4fe20b201760 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54af00c9f7384f5c886d4fe20b201760 2024-11-09T21:17:08,425 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54af00c9f7384f5c886d4fe20b201760, entries=150, sequenceid=1051, filesize=12.0 K 2024-11-09T21:17:08,426 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c7ac4dbed742a563978c6ff5cf7dc21b in 1263ms, sequenceid=1051, compaction requested=true 2024-11-09T21:17:08,427 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/6d9194f102ca4556a3285721eb49a82a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3778005ebd5843ed95fc4032614c3deb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5af3a23aac9445f4980b54c28c75908a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/2bce6518876c43b1a3cae4aa1fd76741, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/46ab52e8da22432a958f47f0209d4e05, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/1c8fb74cef594300a8a71969ece851d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/627f2d64855544458ec9b6a9e365579c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3df8470c0bb341208ca821d8b8de3c92, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3f1fd686134e4c20a4f28ccb78f09b99, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/daceb7acecd6431da788ff5191f41d13, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/81877cb88f2c4f16ae30fed83d66ccc2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9bc4b82d13ad47e1aeab5f5e55f8c474, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/49cce14f031845719dd4d15ea5ab56be, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f8f5734786764a4aadd9c473c1a84bfe, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b059b85079444fd8bfccb30c210a6f20, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b1dd3d74ee4c4b128d6ebb60ce02e229, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/965246391be943d2b31fb61b478ebfbc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/503b7f3b99aa40c9846269054db7ac3a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/ffa53238958247c8a6c65c9ef7044e2e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/a5b8b40851ef4d869b87e0e47b5f1542, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/97428cfa74e34619a55ffc85508a2bed, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7ffd9baff3ab471baff77d1166822f23, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f1a748b482904aa296b2e51d81dda854, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/70b9aee494874cbd82c629488ceb572f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/50be73876534449998df198303e1b935, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/64704cd33e664ea48fe15582a5276d07, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/83ae40bffa594d51b0addc1d1c456c4a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5b736a228f164dd8b554bc1c8c85bb6a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b3482dabb5f8455bb2fa6618d789e7e7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/bbfa39e5b8e04234adef33082cffde6e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/48a879b14c5542f69257a4b3dd2ca102, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7a1fed30fc9c4911940b7e18292864e4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e25e980c745249d395a93ed9ada68600, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3d9420f2ff0a4fe381882438c4b6c880] to archive 2024-11-09T21:17:08,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:17:08,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/6d9194f102ca4556a3285721eb49a82a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/6d9194f102ca4556a3285721eb49a82a 2024-11-09T21:17:08,432 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3778005ebd5843ed95fc4032614c3deb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3778005ebd5843ed95fc4032614c3deb 2024-11-09T21:17:08,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5af3a23aac9445f4980b54c28c75908a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5af3a23aac9445f4980b54c28c75908a 2024-11-09T21:17:08,435 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/2bce6518876c43b1a3cae4aa1fd76741 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/2bce6518876c43b1a3cae4aa1fd76741 2024-11-09T21:17:08,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/46ab52e8da22432a958f47f0209d4e05 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/46ab52e8da22432a958f47f0209d4e05 2024-11-09T21:17:08,438 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/1c8fb74cef594300a8a71969ece851d1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/1c8fb74cef594300a8a71969ece851d1 2024-11-09T21:17:08,440 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/627f2d64855544458ec9b6a9e365579c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/627f2d64855544458ec9b6a9e365579c 2024-11-09T21:17:08,441 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3df8470c0bb341208ca821d8b8de3c92 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3df8470c0bb341208ca821d8b8de3c92 2024-11-09T21:17:08,443 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3f1fd686134e4c20a4f28ccb78f09b99 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3f1fd686134e4c20a4f28ccb78f09b99 2024-11-09T21:17:08,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/daceb7acecd6431da788ff5191f41d13 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/daceb7acecd6431da788ff5191f41d13 2024-11-09T21:17:08,446 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/81877cb88f2c4f16ae30fed83d66ccc2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/81877cb88f2c4f16ae30fed83d66ccc2 2024-11-09T21:17:08,447 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9bc4b82d13ad47e1aeab5f5e55f8c474 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9bc4b82d13ad47e1aeab5f5e55f8c474 2024-11-09T21:17:08,448 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/49cce14f031845719dd4d15ea5ab56be to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/49cce14f031845719dd4d15ea5ab56be 2024-11-09T21:17:08,449 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f8f5734786764a4aadd9c473c1a84bfe to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f8f5734786764a4aadd9c473c1a84bfe 2024-11-09T21:17:08,451 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b059b85079444fd8bfccb30c210a6f20 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b059b85079444fd8bfccb30c210a6f20 2024-11-09T21:17:08,452 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b1dd3d74ee4c4b128d6ebb60ce02e229 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b1dd3d74ee4c4b128d6ebb60ce02e229 2024-11-09T21:17:08,453 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/965246391be943d2b31fb61b478ebfbc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/965246391be943d2b31fb61b478ebfbc 2024-11-09T21:17:08,455 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/503b7f3b99aa40c9846269054db7ac3a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/503b7f3b99aa40c9846269054db7ac3a 2024-11-09T21:17:08,456 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/ffa53238958247c8a6c65c9ef7044e2e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/ffa53238958247c8a6c65c9ef7044e2e 2024-11-09T21:17:08,457 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/a5b8b40851ef4d869b87e0e47b5f1542 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/a5b8b40851ef4d869b87e0e47b5f1542 2024-11-09T21:17:08,458 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/97428cfa74e34619a55ffc85508a2bed to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/97428cfa74e34619a55ffc85508a2bed 2024-11-09T21:17:08,459 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7ffd9baff3ab471baff77d1166822f23 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7ffd9baff3ab471baff77d1166822f23 2024-11-09T21:17:08,461 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f1a748b482904aa296b2e51d81dda854 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/f1a748b482904aa296b2e51d81dda854 2024-11-09T21:17:08,462 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/70b9aee494874cbd82c629488ceb572f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/70b9aee494874cbd82c629488ceb572f 2024-11-09T21:17:08,463 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/50be73876534449998df198303e1b935 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/50be73876534449998df198303e1b935 2024-11-09T21:17:08,465 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/64704cd33e664ea48fe15582a5276d07 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/64704cd33e664ea48fe15582a5276d07 2024-11-09T21:17:08,466 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/83ae40bffa594d51b0addc1d1c456c4a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/83ae40bffa594d51b0addc1d1c456c4a 2024-11-09T21:17:08,467 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5b736a228f164dd8b554bc1c8c85bb6a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/5b736a228f164dd8b554bc1c8c85bb6a 2024-11-09T21:17:08,469 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b3482dabb5f8455bb2fa6618d789e7e7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/b3482dabb5f8455bb2fa6618d789e7e7 2024-11-09T21:17:08,470 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/bbfa39e5b8e04234adef33082cffde6e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/bbfa39e5b8e04234adef33082cffde6e 2024-11-09T21:17:08,471 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/48a879b14c5542f69257a4b3dd2ca102 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/48a879b14c5542f69257a4b3dd2ca102 2024-11-09T21:17:08,473 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7a1fed30fc9c4911940b7e18292864e4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/7a1fed30fc9c4911940b7e18292864e4 2024-11-09T21:17:08,474 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e25e980c745249d395a93ed9ada68600 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e25e980c745249d395a93ed9ada68600 2024-11-09T21:17:08,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3d9420f2ff0a4fe381882438c4b6c880 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/3d9420f2ff0a4fe381882438c4b6c880 2024-11-09T21:17:08,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d82d320109844ef58399413f984af2ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0c1a87ad8ca643038e9c62d499aac003, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b3e433ca19674f9e959c2c830df93a6e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1fb1fdd1a11e4f6593d4230b60e3a449, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/95a288c64bfe4dbdbc8cafa572ec9ba4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/c991cab9f5ec4a569d531470b7955716, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/a874f49e3a7445868448ee56455db7dd, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/400493f7d9404ff38f2eefc76c1f8dac, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bb6ed32337294a6a9f7f522a3f100dfb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/3af8b24f47344f37b752c79847e33caf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0acb4191d9464e8bb51da440fdb1f78a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/abece10ad3a942d88656e9fd1fe66f67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/795d717f2e674faeaec810c830065e7e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d8a424631be14028b437f97ca34d21c8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/ae0621505a754ea18941234521a57599, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9e11ff2d0ab14adfa31088821228662a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/86c703edca644aa7ba3d11ae8d34bd5d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bdbd4077d46b40c8a86d571126cecf8f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b79f0f6b4bdc482ea89b9a44f4d13ed5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/59bb252b823745ff9b1b0e10f65bb5ad, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/11e3f5f8e2fe4306b75460ca4b96ff3b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0364ec03c91f4be4881de5769618e316, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/23311cef913d4f378395866d67fab26c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d583e47449484145926ba5657af4874c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4a3367a12f2440258ebed54e3b1b43d7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/7f1fc529eae84b9fb5327513f4ae3704, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9dee42f2c87f4b39bda29988857a099b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/418affb6d64a49af921772c8d98666b8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/be588b32c8fe4831a0d19b3d6ff3ef47, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/72586780ab114d3c880274c77ec020e9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b44a23a6825c438891dff59ba29dacb8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/5a470733833141ae98f9894f696d9b94, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1387a7d7ea864aae96b50d8f19071448, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/8aa1b86dd10f472ba38da534059ed179] to archive 2024-11-09T21:17:08,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:17:08,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d82d320109844ef58399413f984af2ef to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d82d320109844ef58399413f984af2ef 2024-11-09T21:17:08,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0c1a87ad8ca643038e9c62d499aac003 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0c1a87ad8ca643038e9c62d499aac003 2024-11-09T21:17:08,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b3e433ca19674f9e959c2c830df93a6e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b3e433ca19674f9e959c2c830df93a6e 2024-11-09T21:17:08,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1fb1fdd1a11e4f6593d4230b60e3a449 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1fb1fdd1a11e4f6593d4230b60e3a449 2024-11-09T21:17:08,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/95a288c64bfe4dbdbc8cafa572ec9ba4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/95a288c64bfe4dbdbc8cafa572ec9ba4 2024-11-09T21:17:08,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/c991cab9f5ec4a569d531470b7955716 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/c991cab9f5ec4a569d531470b7955716 2024-11-09T21:17:08,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/a874f49e3a7445868448ee56455db7dd to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/a874f49e3a7445868448ee56455db7dd 2024-11-09T21:17:08,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/400493f7d9404ff38f2eefc76c1f8dac to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/400493f7d9404ff38f2eefc76c1f8dac 2024-11-09T21:17:08,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bb6ed32337294a6a9f7f522a3f100dfb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bb6ed32337294a6a9f7f522a3f100dfb 2024-11-09T21:17:08,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/3af8b24f47344f37b752c79847e33caf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/3af8b24f47344f37b752c79847e33caf 2024-11-09T21:17:08,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0acb4191d9464e8bb51da440fdb1f78a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0acb4191d9464e8bb51da440fdb1f78a 2024-11-09T21:17:08,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/abece10ad3a942d88656e9fd1fe66f67 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/abece10ad3a942d88656e9fd1fe66f67 2024-11-09T21:17:08,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/795d717f2e674faeaec810c830065e7e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/795d717f2e674faeaec810c830065e7e 2024-11-09T21:17:08,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d8a424631be14028b437f97ca34d21c8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d8a424631be14028b437f97ca34d21c8 2024-11-09T21:17:08,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/ae0621505a754ea18941234521a57599 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/ae0621505a754ea18941234521a57599 2024-11-09T21:17:08,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9e11ff2d0ab14adfa31088821228662a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9e11ff2d0ab14adfa31088821228662a 2024-11-09T21:17:08,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/86c703edca644aa7ba3d11ae8d34bd5d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/86c703edca644aa7ba3d11ae8d34bd5d 2024-11-09T21:17:08,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bdbd4077d46b40c8a86d571126cecf8f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/bdbd4077d46b40c8a86d571126cecf8f 2024-11-09T21:17:08,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b79f0f6b4bdc482ea89b9a44f4d13ed5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b79f0f6b4bdc482ea89b9a44f4d13ed5 2024-11-09T21:17:08,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/59bb252b823745ff9b1b0e10f65bb5ad to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/59bb252b823745ff9b1b0e10f65bb5ad 2024-11-09T21:17:08,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/11e3f5f8e2fe4306b75460ca4b96ff3b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/11e3f5f8e2fe4306b75460ca4b96ff3b 2024-11-09T21:17:08,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0364ec03c91f4be4881de5769618e316 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/0364ec03c91f4be4881de5769618e316 2024-11-09T21:17:08,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/23311cef913d4f378395866d67fab26c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/23311cef913d4f378395866d67fab26c 2024-11-09T21:17:08,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d583e47449484145926ba5657af4874c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/d583e47449484145926ba5657af4874c 2024-11-09T21:17:08,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4a3367a12f2440258ebed54e3b1b43d7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4a3367a12f2440258ebed54e3b1b43d7 2024-11-09T21:17:08,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/7f1fc529eae84b9fb5327513f4ae3704 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/7f1fc529eae84b9fb5327513f4ae3704 2024-11-09T21:17:08,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9dee42f2c87f4b39bda29988857a099b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/9dee42f2c87f4b39bda29988857a099b 2024-11-09T21:17:08,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/418affb6d64a49af921772c8d98666b8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/418affb6d64a49af921772c8d98666b8 2024-11-09T21:17:08,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/be588b32c8fe4831a0d19b3d6ff3ef47 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/be588b32c8fe4831a0d19b3d6ff3ef47 2024-11-09T21:17:08,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/72586780ab114d3c880274c77ec020e9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/72586780ab114d3c880274c77ec020e9 2024-11-09T21:17:08,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b44a23a6825c438891dff59ba29dacb8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/b44a23a6825c438891dff59ba29dacb8 2024-11-09T21:17:08,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/5a470733833141ae98f9894f696d9b94 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/5a470733833141ae98f9894f696d9b94 2024-11-09T21:17:08,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1387a7d7ea864aae96b50d8f19071448 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/1387a7d7ea864aae96b50d8f19071448 2024-11-09T21:17:08,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/8aa1b86dd10f472ba38da534059ed179 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/8aa1b86dd10f472ba38da534059ed179 2024-11-09T21:17:08,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/05b54913b1564db89bbce3ae77105dd0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/46bb94ec4f644fc98ba70fb07ab08c7e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8e34ab30e43646f1b79e0443fbc44b8c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a0048701cf9b459fbe5f72253869a0b8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/537529b1979947b68037d41567f6c4bf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9979ebb76bc44b958156b75a2ce9db7f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/f63a91634edd4f7791e53e7ef2cfd9ad, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8c1ab557fc57456b8e91b704e00d9828, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/3f461f641c1346f8bf746856b9a7445b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54027cdfab8f40ce8e988ec50acee4db, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/644f3df39f154f2db952d4afa2bbbd73, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/0959552a6fca4b5babe87c3622c42a87, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a728c7764b374182a74e27ce64a7eea0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/38f2227bd4774021beca904a975e0161, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/798f64b0649446fabda49cf7e4b36d6a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6202a37e56db4755ab0ec7933cb7e9ad, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6dee2330cf83435ebf7ef20270634eeb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/d6390ca6eed44d21b089b2101f0b00e2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fc116322578478f92df8decaac2e55c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/7b34abcd428d495a8a03249362988722, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fbd8098fcb0430e95699f28eebb89f1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ac524e2cf32e43ae870068a0779a0fce, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8110cc189592493d8a8ef7ef80692fae, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/bbedc8600e3c49bc943b8b2c62d13f04, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a6eacd6efdc4427983e32635b8ac0366, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6c476b494382441cb525608bec5637ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9bc84794d40641e2bef56a3515646eb1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ccfe23e378e24af6b0a91b8575a589da, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5677ce726d56423293a0a5fa7d9e41f5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/833debfc82104d7b82829b4078d0021f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4ac2ca3472c34a2d910f0df3c2fa6f56, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/50ef30eaa4864904b87cb41117364700, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/c22120763dd14a4ab3528c6a042d2a3b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/dc31e68c78a64ba680f88339cce61090] to archive 2024-11-09T21:17:08,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:17:08,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/05b54913b1564db89bbce3ae77105dd0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/05b54913b1564db89bbce3ae77105dd0 2024-11-09T21:17:08,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/46bb94ec4f644fc98ba70fb07ab08c7e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/46bb94ec4f644fc98ba70fb07ab08c7e 2024-11-09T21:17:08,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8e34ab30e43646f1b79e0443fbc44b8c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8e34ab30e43646f1b79e0443fbc44b8c 2024-11-09T21:17:08,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a0048701cf9b459fbe5f72253869a0b8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a0048701cf9b459fbe5f72253869a0b8 2024-11-09T21:17:08,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/537529b1979947b68037d41567f6c4bf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/537529b1979947b68037d41567f6c4bf 2024-11-09T21:17:08,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9979ebb76bc44b958156b75a2ce9db7f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9979ebb76bc44b958156b75a2ce9db7f 2024-11-09T21:17:08,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/f63a91634edd4f7791e53e7ef2cfd9ad to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/f63a91634edd4f7791e53e7ef2cfd9ad 2024-11-09T21:17:08,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8c1ab557fc57456b8e91b704e00d9828 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8c1ab557fc57456b8e91b704e00d9828 2024-11-09T21:17:08,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/3f461f641c1346f8bf746856b9a7445b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/3f461f641c1346f8bf746856b9a7445b 2024-11-09T21:17:08,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54027cdfab8f40ce8e988ec50acee4db to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54027cdfab8f40ce8e988ec50acee4db 2024-11-09T21:17:08,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/644f3df39f154f2db952d4afa2bbbd73 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/644f3df39f154f2db952d4afa2bbbd73 2024-11-09T21:17:08,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/0959552a6fca4b5babe87c3622c42a87 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/0959552a6fca4b5babe87c3622c42a87 2024-11-09T21:17:08,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a728c7764b374182a74e27ce64a7eea0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a728c7764b374182a74e27ce64a7eea0 2024-11-09T21:17:08,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/38f2227bd4774021beca904a975e0161 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/38f2227bd4774021beca904a975e0161 2024-11-09T21:17:08,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/798f64b0649446fabda49cf7e4b36d6a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/798f64b0649446fabda49cf7e4b36d6a 2024-11-09T21:17:08,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6202a37e56db4755ab0ec7933cb7e9ad to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6202a37e56db4755ab0ec7933cb7e9ad 2024-11-09T21:17:08,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6dee2330cf83435ebf7ef20270634eeb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6dee2330cf83435ebf7ef20270634eeb 2024-11-09T21:17:08,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/d6390ca6eed44d21b089b2101f0b00e2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/d6390ca6eed44d21b089b2101f0b00e2 2024-11-09T21:17:08,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fc116322578478f92df8decaac2e55c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fc116322578478f92df8decaac2e55c 2024-11-09T21:17:08,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/7b34abcd428d495a8a03249362988722 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/7b34abcd428d495a8a03249362988722 2024-11-09T21:17:08,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fbd8098fcb0430e95699f28eebb89f1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5fbd8098fcb0430e95699f28eebb89f1 2024-11-09T21:17:08,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ac524e2cf32e43ae870068a0779a0fce to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ac524e2cf32e43ae870068a0779a0fce 2024-11-09T21:17:08,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8110cc189592493d8a8ef7ef80692fae to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/8110cc189592493d8a8ef7ef80692fae 2024-11-09T21:17:08,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/bbedc8600e3c49bc943b8b2c62d13f04 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/bbedc8600e3c49bc943b8b2c62d13f04 2024-11-09T21:17:08,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a6eacd6efdc4427983e32635b8ac0366 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/a6eacd6efdc4427983e32635b8ac0366 2024-11-09T21:17:08,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6c476b494382441cb525608bec5637ef to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/6c476b494382441cb525608bec5637ef 2024-11-09T21:17:08,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9bc84794d40641e2bef56a3515646eb1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/9bc84794d40641e2bef56a3515646eb1 2024-11-09T21:17:08,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ccfe23e378e24af6b0a91b8575a589da to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/ccfe23e378e24af6b0a91b8575a589da 2024-11-09T21:17:08,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5677ce726d56423293a0a5fa7d9e41f5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/5677ce726d56423293a0a5fa7d9e41f5 2024-11-09T21:17:08,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/833debfc82104d7b82829b4078d0021f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/833debfc82104d7b82829b4078d0021f 2024-11-09T21:17:08,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4ac2ca3472c34a2d910f0df3c2fa6f56 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4ac2ca3472c34a2d910f0df3c2fa6f56 2024-11-09T21:17:08,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/50ef30eaa4864904b87cb41117364700 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/50ef30eaa4864904b87cb41117364700 2024-11-09T21:17:08,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/c22120763dd14a4ab3528c6a042d2a3b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/c22120763dd14a4ab3528c6a042d2a3b 2024-11-09T21:17:08,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/dc31e68c78a64ba680f88339cce61090 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/dc31e68c78a64ba680f88339cce61090 2024-11-09T21:17:08,561 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/recovered.edits/1054.seqid, newMaxSeqId=1054, maxSeqId=1 2024-11-09T21:17:08,561 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. 2024-11-09T21:17:08,561 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for c7ac4dbed742a563978c6ff5cf7dc21b: Waiting for close lock at 1731187027162Running coprocessor pre-close hooks at 1731187027162Disabling compacts and flushes for region at 1731187027162Disabling writes for close at 1731187027163 (+1 ms)Obtaining lock to block concurrent updates at 1731187027163Preparing flush snapshotting stores in c7ac4dbed742a563978c6ff5cf7dc21b at 1731187027163Finished memstore snapshotting TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731187027164 (+1 ms)Flushing stores of TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b. at 1731187027166 (+2 ms)Flushing c7ac4dbed742a563978c6ff5cf7dc21b/A: creating writer at 1731187027166Flushing c7ac4dbed742a563978c6ff5cf7dc21b/A: appending metadata at 1731187027171 (+5 ms)Flushing c7ac4dbed742a563978c6ff5cf7dc21b/A: closing flushed file at 1731187027171Flushing c7ac4dbed742a563978c6ff5cf7dc21b/B: creating writer at 1731187027586 (+415 ms)Flushing c7ac4dbed742a563978c6ff5cf7dc21b/B: appending metadata at 1731187027590 (+4 ms)Flushing c7ac4dbed742a563978c6ff5cf7dc21b/B: closing flushed file at 1731187027590Flushing c7ac4dbed742a563978c6ff5cf7dc21b/C: creating writer at 1731187027999 (+409 ms)Flushing c7ac4dbed742a563978c6ff5cf7dc21b/C: appending metadata at 1731187028002 (+3 ms)Flushing c7ac4dbed742a563978c6ff5cf7dc21b/C: closing flushed file at 1731187028002Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@382fe1ff: reopening flushed file at 1731187028410 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34f214c5: reopening flushed file at 1731187028415 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61453264: reopening flushed file at 1731187028420 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for c7ac4dbed742a563978c6ff5cf7dc21b in 1263ms, sequenceid=1051, compaction requested=true at 1731187028426 (+6 ms)Writing region close event to WAL at 1731187028558 (+132 ms)Running coprocessor post-close hooks at 1731187028561 (+3 ms)Closed at 1731187028561 2024-11-09T21:17:08,563 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:17:08,564 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=c7ac4dbed742a563978c6ff5cf7dc21b, regionState=CLOSED 2024-11-09T21:17:08,565 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:17:08,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-09T21:17:08,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure c7ac4dbed742a563978c6ff5cf7dc21b, server=9514d6d5e5be,38413,1731186934857 in 1.5580 sec 2024-11-09T21:17:08,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-09T21:17:08,569 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=c7ac4dbed742a563978c6ff5cf7dc21b, UNASSIGN in 1.5630 sec 2024-11-09T21:17:08,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-09T21:17:08,571 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5670 sec 2024-11-09T21:17:08,572 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187028572"}]},"ts":"1731187028572"} 2024-11-09T21:17:08,573 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T21:17:08,573 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T21:17:08,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5790 sec 2024-11-09T21:17:09,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-09T21:17:09,148 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:09,149 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T21:17:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,154 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-09T21:17:09,156 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,161 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:17:09,165 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/recovered.edits] 2024-11-09T21:17:09,212 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9255b68fbb5c43cd851de16187bbe702 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/9255b68fbb5c43cd851de16187bbe702 2024-11-09T21:17:09,214 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/c754ed41cda344da9da3b6399a9fceba to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/c754ed41cda344da9da3b6399a9fceba 2024-11-09T21:17:09,215 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e61f9e10b49a4cd49b51cf01e9f434be to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/A/e61f9e10b49a4cd49b51cf01e9f434be 2024-11-09T21:17:09,216 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4340055ac0be4789a48d08368443130f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/4340055ac0be4789a48d08368443130f 2024-11-09T21:17:09,218 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/6bfa2c9eb0ad4e8db123a4c56303bc9d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/6bfa2c9eb0ad4e8db123a4c56303bc9d 2024-11-09T21:17:09,219 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/da40fd3a781a4f2dafab5153012a6659 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/B/da40fd3a781a4f2dafab5153012a6659 2024-11-09T21:17:09,221 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/26eb34684d4d4af2b0d8b7a3d1662450 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/26eb34684d4d4af2b0d8b7a3d1662450 2024-11-09T21:17:09,222 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4bb8668e52b04302991482b331de64a2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/4bb8668e52b04302991482b331de64a2 2024-11-09T21:17:09,223 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54af00c9f7384f5c886d4fe20b201760 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/C/54af00c9f7384f5c886d4fe20b201760 2024-11-09T21:17:09,226 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/recovered.edits/1054.seqid to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b/recovered.edits/1054.seqid 2024-11-09T21:17:09,227 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/c7ac4dbed742a563978c6ff5cf7dc21b 2024-11-09T21:17:09,227 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T21:17:09,229 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,233 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T21:17:09,236 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T21:17:09,237 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,237 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T21:17:09,237 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731187029237"}]},"ts":"9223372036854775807"} 2024-11-09T21:17:09,240 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T21:17:09,240 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => c7ac4dbed742a563978c6ff5cf7dc21b, NAME => 'TestAcidGuarantees,,1731186995021.c7ac4dbed742a563978c6ff5cf7dc21b.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T21:17:09,240 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T21:17:09,240 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731187029240"}]},"ts":"9223372036854775807"} 2024-11-09T21:17:09,242 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T21:17:09,243 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,245 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 94 msec 2024-11-09T21:17:09,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-09T21:17:09,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T21:17:09,267 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:09,277 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 252), OpenFileDescriptor=447 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 402), ProcessCount=11 (was 11), AvailableMemoryMB=2983 (was 3074) 2024-11-09T21:17:09,286 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=2983 2024-11-09T21:17:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:17:09,288 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:17:09,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:09,290 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:17:09,290 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:09,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-09T21:17:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T21:17:09,291 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:17:09,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742170_1346 (size=1061) 2024-11-09T21:17:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T21:17:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T21:17:09,698 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:17:09,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742171_1347 (size=53) 2024-11-09T21:17:09,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T21:17:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:17:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 010ea70b10f616a09e8971f27ee90066, disabling compactions & flushes 2024-11-09T21:17:10,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. after waiting 0 ms 2024-11-09T21:17:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,103 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,103 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 010ea70b10f616a09e8971f27ee90066: Waiting for close lock at 1731187030103Disabling compacts and flushes for region at 1731187030103Disabling writes for close at 1731187030103Writing region close event to WAL at 1731187030103Closed at 1731187030103 2024-11-09T21:17:10,104 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:17:10,105 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731187030104"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187030104"}]},"ts":"1731187030104"} 2024-11-09T21:17:10,107 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:17:10,108 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:17:10,108 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187030108"}]},"ts":"1731187030108"} 2024-11-09T21:17:10,109 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T21:17:10,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, ASSIGN}] 2024-11-09T21:17:10,110 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, ASSIGN 2024-11-09T21:17:10,111 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, ASSIGN; state=OFFLINE, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=false 2024-11-09T21:17:10,262 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:10,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, ASSIGN because future has completed 2024-11-09T21:17:10,268 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:17:10,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T21:17:10,428 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,428 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:17:10,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:17:10,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,429 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,431 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,433 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:10,433 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 010ea70b10f616a09e8971f27ee90066 columnFamilyName A 2024-11-09T21:17:10,433 DEBUG [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:10,434 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(327): Store=010ea70b10f616a09e8971f27ee90066/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:10,434 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,435 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:10,435 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 010ea70b10f616a09e8971f27ee90066 columnFamilyName B 2024-11-09T21:17:10,436 DEBUG [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:10,436 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(327): Store=010ea70b10f616a09e8971f27ee90066/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:10,436 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,438 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:10,438 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 010ea70b10f616a09e8971f27ee90066 columnFamilyName C 2024-11-09T21:17:10,438 DEBUG [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:10,439 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(327): Store=010ea70b10f616a09e8971f27ee90066/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:10,439 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,439 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,440 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,440 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,441 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,441 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,441 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:17:10,442 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,443 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:17:10,444 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened 010ea70b10f616a09e8971f27ee90066; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63103687, jitterRate=-0.059681788086891174}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:17:10,444 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:10,444 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for 010ea70b10f616a09e8971f27ee90066: Running coprocessor pre-open hook at 1731187030429Writing region info on filesystem at 1731187030429Initializing all the Stores at 1731187030430 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187030430Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187030431 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187030431Cleaning up temporary data from old regions at 1731187030441 (+10 ms)Running coprocessor post-open hooks at 1731187030444 (+3 ms)Region opened successfully at 1731187030444 2024-11-09T21:17:10,445 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., pid=93, masterSystemTime=1731187030421 2024-11-09T21:17:10,446 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,446 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:10,447 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:10,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:17:10,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-09T21:17:10,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 in 181 msec 2024-11-09T21:17:10,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-09T21:17:10,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, ASSIGN in 341 msec 2024-11-09T21:17:10,452 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:17:10,452 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187030452"}]},"ts":"1731187030452"} 2024-11-09T21:17:10,453 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T21:17:10,453 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:17:10,454 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1650 sec 2024-11-09T21:17:11,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-09T21:17:11,436 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:11,437 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ab4b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:11,437 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:11,438 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:11,439 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16e5c835, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:11,439 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:11,440 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38784, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:11,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@108975d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:11,441 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:11,442 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:11,443 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:11,444 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:17:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:17:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@35359423 2024-11-09T21:17:11,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:17:11,445 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:17:11,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:17:11,447 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:17:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:11,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742172_1348 (size=1097) 2024-11-09T21:17:11,859 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T21:17:11,859 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T21:17:11,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:17:11,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, REOPEN/MOVE}] 2024-11-09T21:17:11,865 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, REOPEN/MOVE 2024-11-09T21:17:11,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:11,867 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, REOPEN/MOVE because future has completed 2024-11-09T21:17:11,868 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:17:11,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:17:12,021 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,021 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:17:12,021 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing 010ea70b10f616a09e8971f27ee90066, disabling compactions & flushes 2024-11-09T21:17:12,021 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,021 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,021 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. after waiting 0 ms 2024-11-09T21:17:12,021 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,025 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:17:12,026 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,026 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for 010ea70b10f616a09e8971f27ee90066: Waiting for close lock at 1731187032021Running coprocessor pre-close hooks at 1731187032021Disabling compacts and flushes for region at 1731187032021Disabling writes for close at 1731187032021Writing region close event to WAL at 1731187032021Running coprocessor post-close hooks at 1731187032026 (+5 ms)Closed at 1731187032026 2024-11-09T21:17:12,026 WARN [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: 010ea70b10f616a09e8971f27ee90066 to self. 2024-11-09T21:17:12,028 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,029 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=CLOSED 2024-11-09T21:17:12,030 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:17:12,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-09T21:17:12,035 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 in 164 msec 2024-11-09T21:17:12,036 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, REOPEN/MOVE; state=CLOSED, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=true 2024-11-09T21:17:12,187 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,191 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, REOPEN/MOVE because future has completed 2024-11-09T21:17:12,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:17:12,349 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,349 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:17:12,349 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,350 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:17:12,350 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,350 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,351 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,352 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:12,352 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 010ea70b10f616a09e8971f27ee90066 columnFamilyName A 2024-11-09T21:17:12,353 DEBUG [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:12,353 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(327): Store=010ea70b10f616a09e8971f27ee90066/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:12,353 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,354 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:12,354 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 010ea70b10f616a09e8971f27ee90066 columnFamilyName B 2024-11-09T21:17:12,354 DEBUG [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:12,354 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(327): Store=010ea70b10f616a09e8971f27ee90066/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:12,355 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,355 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:12,355 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 010ea70b10f616a09e8971f27ee90066 columnFamilyName C 2024-11-09T21:17:12,355 DEBUG [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:12,355 INFO [StoreOpener-010ea70b10f616a09e8971f27ee90066-1 {}] regionserver.HStore(327): Store=010ea70b10f616a09e8971f27ee90066/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:12,355 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,356 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,356 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,357 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,358 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,358 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,358 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:17:12,359 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,360 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened 010ea70b10f616a09e8971f27ee90066; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61759679, jitterRate=-0.07970906794071198}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:17:12,360 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,360 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for 010ea70b10f616a09e8971f27ee90066: Running coprocessor pre-open hook at 1731187032350Writing region info on filesystem at 1731187032350Initializing all the Stores at 1731187032350Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187032350Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187032351 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187032351Cleaning up temporary data from old regions at 1731187032358 (+7 ms)Running coprocessor post-open hooks at 1731187032360 (+2 ms)Region opened successfully at 1731187032360 2024-11-09T21:17:12,361 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., pid=98, masterSystemTime=1731187032347 2024-11-09T21:17:12,362 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,363 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,363 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=OPEN, openSeqNum=5, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,364 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:17:12,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-09T21:17:12,367 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 in 173 msec 2024-11-09T21:17:12,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-09T21:17:12,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, REOPEN/MOVE in 503 msec 2024-11-09T21:17:12,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-09T21:17:12,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 508 msec 2024-11-09T21:17:12,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 926 msec 2024-11-09T21:17:12,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-09T21:17:12,375 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:12,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173aa1ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,377 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7728bbe5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,377 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,378 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,378 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42954, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,379 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295ec074, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d5e8400, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c7c603, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,381 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42986, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,382 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16fea0ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ce5c6bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,384 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c35cad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,384 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,385 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,385 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50e8334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,386 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b7fbdbe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,386 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15a87b6f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,387 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,388 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43032, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8eca8d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,390 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37842ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,390 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,390 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,391 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c7f200d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,391 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,392 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43058, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b3aa8ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,394 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bbf44a1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,394 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,394 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,395 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fa93b57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,395 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,396 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43076, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,397 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6750b0f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ad32ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,398 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f2a2542, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,399 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43088, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,399 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@560654dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,400 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@615de503, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,400 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,400 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,400 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1104e30c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,402 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@643d225b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,403 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5690e3aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,403 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,403 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,403 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,403 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70d13b02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,404 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,404 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43110, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,405 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32dea889, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,405 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a170ebf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,405 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:12,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:12,406 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f07f9c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:12,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:12,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:12,407 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43140, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:12,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ac4f51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:12,408 DEBUG [Thread-1599 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,408 DEBUG [Thread-1595 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,408 DEBUG [Thread-1601 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,408 DEBUG [Thread-1593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,408 DEBUG [Thread-1597 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,408 DEBUG [Thread-1584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,408 DEBUG [Thread-1586 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,409 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56168, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,410 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56202, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56218, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,411 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56224, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,413 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,414 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:12,414 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,415 DEBUG [Thread-1588 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,415 DEBUG [Thread-1582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,416 DEBUG [Thread-1588 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,416 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,417 DEBUG [Thread-1588 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-09T21:17:12,417 DEBUG [Thread-1590 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T21:17:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56232, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:12,418 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56242, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56246, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:12,419 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:12,420 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:12,420 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,421 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,422 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,423 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,425 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:17:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,457 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:17:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:12,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:12,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110955fe2d64e36a4046ade1cada9c5021f2_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187032415/Put/seqid=0 2024-11-09T21:17:12,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742173_1349 (size=17034) 2024-11-09T21:17:12,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187032478, max timestamp=1731187032494Num uniques 859; 2024-11-09T21:17:12,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:17:12,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187032478, max timestamp=1731187032494Num uniques 859; 2024-11-09T21:17:12,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:17:12,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731187032478, max timestamp=1731187032494Num uniques 859; 2024-11-09T21:17:12,498 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-09T21:17:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T21:17:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187092521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,538 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187092533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,539 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187092534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,548 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187092535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187092538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T21:17:12,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:12,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187092646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187092646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,651 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187092657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187092657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,665 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187092657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T21:17:12,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:12,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T21:17:12,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187092857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187092857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T21:17:12,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187092876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:12,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:12,881 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,881 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187092876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187092876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:12,881 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:12,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:12,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:12,893 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110955fe2d64e36a4046ade1cada9c5021f2_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110955fe2d64e36a4046ade1cada9c5021f2_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:12,894 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/acaf6075485e40d7964ad856b1c2ad31, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:12,895 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/acaf6075485e40d7964ad856b1c2ad31 is 175, key is test_row_0/A:col10/1731187032415/Put/seqid=0 2024-11-09T21:17:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742174_1350 (size=48139) 2024-11-09T21:17:12,900 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/acaf6075485e40d7964ad856b1c2ad31 2024-11-09T21:17:12,925 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a87d65ef3c4f44aab3b01127abd403c1 is 50, key is test_row_0/B:col10/1731187032415/Put/seqid=0 2024-11-09T21:17:12,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742175_1351 (size=12001) 2024-11-09T21:17:12,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a87d65ef3c4f44aab3b01127abd403c1 2024-11-09T21:17:12,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/298f4da6891e4ce58be3b32b6fb70d35 is 50, key is test_row_0/C:col10/1731187032415/Put/seqid=0 2024-11-09T21:17:12,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742176_1352 (size=12001) 2024-11-09T21:17:12,965 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/298f4da6891e4ce58be3b32b6fb70d35 2024-11-09T21:17:12,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/acaf6075485e40d7964ad856b1c2ad31 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31 2024-11-09T21:17:12,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31, entries=250, sequenceid=15, filesize=47.0 K 2024-11-09T21:17:12,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a87d65ef3c4f44aab3b01127abd403c1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a87d65ef3c4f44aab3b01127abd403c1 2024-11-09T21:17:12,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a87d65ef3c4f44aab3b01127abd403c1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T21:17:12,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/298f4da6891e4ce58be3b32b6fb70d35 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/298f4da6891e4ce58be3b32b6fb70d35 2024-11-09T21:17:12,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/298f4da6891e4ce58be3b32b6fb70d35, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T21:17:12,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 010ea70b10f616a09e8971f27ee90066 in 529ms, sequenceid=15, compaction requested=false 2024-11-09T21:17:12,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:13,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-09T21:17:13,033 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:13,034 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-09T21:17:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:13,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:13,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a10bffff5dab48699f251c1afc720c3d_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187032532/Put/seqid=0 2024-11-09T21:17:13,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742177_1353 (size=12154) 2024-11-09T21:17:13,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T21:17:13,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:13,174 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:13,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187093192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187093193, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,210 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187093195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,211 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187093196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187093201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187093307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187093316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187093316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187093316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187093316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:13,449 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a10bffff5dab48699f251c1afc720c3d_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a10bffff5dab48699f251c1afc720c3d_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:13,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/8fe3f2e4bb414ac087d599d0a1ce02a3, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:13,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/8fe3f2e4bb414ac087d599d0a1ce02a3 is 175, key is test_row_0/A:col10/1731187032532/Put/seqid=0 2024-11-09T21:17:13,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742178_1354 (size=30955) 2024-11-09T21:17:13,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/8fe3f2e4bb414ac087d599d0a1ce02a3 2024-11-09T21:17:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a7e3feb3e406491ab27303312bd27dcc is 50, key is test_row_0/B:col10/1731187032532/Put/seqid=0 2024-11-09T21:17:13,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742179_1355 (size=12001) 2024-11-09T21:17:13,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187093516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187093526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187093526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187093526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187093526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,529 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T21:17:13,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187093836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187093836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187093836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187093836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187093836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:13,845 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:13,864 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a7e3feb3e406491ab27303312bd27dcc 2024-11-09T21:17:13,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/6efb403b53534b9cb24bd0041aeab76b is 50, key is test_row_0/C:col10/1731187032532/Put/seqid=0 2024-11-09T21:17:13,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742180_1356 (size=12001) 2024-11-09T21:17:14,091 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:17:14,274 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/6efb403b53534b9cb24bd0041aeab76b 2024-11-09T21:17:14,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/8fe3f2e4bb414ac087d599d0a1ce02a3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3 2024-11-09T21:17:14,285 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3, entries=150, sequenceid=51, filesize=30.2 K 2024-11-09T21:17:14,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a7e3feb3e406491ab27303312bd27dcc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a7e3feb3e406491ab27303312bd27dcc 2024-11-09T21:17:14,290 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a7e3feb3e406491ab27303312bd27dcc, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T21:17:14,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/6efb403b53534b9cb24bd0041aeab76b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6efb403b53534b9cb24bd0041aeab76b 2024-11-09T21:17:14,297 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6efb403b53534b9cb24bd0041aeab76b, entries=150, sequenceid=51, filesize=11.7 K 2024-11-09T21:17:14,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 010ea70b10f616a09e8971f27ee90066 in 1265ms, sequenceid=51, compaction requested=false 2024-11-09T21:17:14,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:14,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:14,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-09T21:17:14,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-09T21:17:14,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-09T21:17:14,307 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8830 sec 2024-11-09T21:17:14,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.8950 sec 2024-11-09T21:17:14,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:14,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T21:17:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:14,365 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:14,372 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b46f36219cb2487bb523aeea1dd54604_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:14,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742181_1357 (size=14594) 2024-11-09T21:17:14,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187094389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,397 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187094394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187094394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,403 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187094396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187094397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187094506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187094506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187094506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,516 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187094516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187094517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,522 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-09T21:17:14,566 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:14,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-09T21:17:14,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:14,568 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:14,569 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:14,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:14,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:14,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:14,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:14,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:14,721 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:14,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:14,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187094726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,735 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187094727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187094727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,736 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187094727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187094727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:14,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:14,786 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:14,789 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b46f36219cb2487bb523aeea1dd54604_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b46f36219cb2487bb523aeea1dd54604_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:14,790 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f7db0c63997f4e1280fc30b8c89739c9, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:14,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f7db0c63997f4e1280fc30b8c89739c9 is 175, key is test_row_0/A:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:14,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742182_1358 (size=39549) 2024-11-09T21:17:14,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:14,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:14,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:14,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:15,026 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:15,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:15,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,027 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187095047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187095047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187095047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187095047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187095047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,051 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,052 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,179 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:15,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:15,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,195 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=66, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f7db0c63997f4e1280fc30b8c89739c9 2024-11-09T21:17:15,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:15,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/9768fb466ee94665bcfc9d79ed71a02e is 50, key is test_row_0/B:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:15,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742183_1359 (size=12001) 2024-11-09T21:17:15,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:15,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:15,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:15,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,484 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187095556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187095556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187095556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187095556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:15,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187095566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:15,572 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:15,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/9768fb466ee94665bcfc9d79ed71a02e 2024-11-09T21:17:15,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/e1337d484f8c487f9b5a7f87a1999039 is 50, key is test_row_0/C:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:15,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742184_1360 (size=12001) 2024-11-09T21:17:15,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/e1337d484f8c487f9b5a7f87a1999039 2024-11-09T21:17:15,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f7db0c63997f4e1280fc30b8c89739c9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9 2024-11-09T21:17:15,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9, entries=200, sequenceid=66, filesize=38.6 K 2024-11-09T21:17:15,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/9768fb466ee94665bcfc9d79ed71a02e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9768fb466ee94665bcfc9d79ed71a02e 2024-11-09T21:17:15,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9768fb466ee94665bcfc9d79ed71a02e, entries=150, sequenceid=66, filesize=11.7 K 2024-11-09T21:17:15,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/e1337d484f8c487f9b5a7f87a1999039 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/e1337d484f8c487f9b5a7f87a1999039 2024-11-09T21:17:15,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:15,637 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,637 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:15,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/e1337d484f8c487f9b5a7f87a1999039, entries=150, sequenceid=66, filesize=11.7 K 2024-11-09T21:17:15,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 010ea70b10f616a09e8971f27ee90066 in 1278ms, sequenceid=66, compaction requested=true 2024-11-09T21:17:15,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:15,641 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:15,642 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:15,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 118643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/A is initiating minor compaction (all files) 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/B is initiating minor compaction (all files) 2024-11-09T21:17:15,646 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/A in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,646 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/B in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,646 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a87d65ef3c4f44aab3b01127abd403c1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a7e3feb3e406491ab27303312bd27dcc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9768fb466ee94665bcfc9d79ed71a02e] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=35.2 K 2024-11-09T21:17:15,646 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=115.9 K 2024-11-09T21:17:15,646 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9] 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a87d65ef3c4f44aab3b01127abd403c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731187032415 2024-11-09T21:17:15,646 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting acaf6075485e40d7964ad856b1c2ad31, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731187032415 2024-11-09T21:17:15,647 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a7e3feb3e406491ab27303312bd27dcc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731187032531 2024-11-09T21:17:15,647 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8fe3f2e4bb414ac087d599d0a1ce02a3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731187032531 2024-11-09T21:17:15,647 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9768fb466ee94665bcfc9d79ed71a02e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731187034360 2024-11-09T21:17:15,647 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7db0c63997f4e1280fc30b8c89739c9, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731187033191 2024-11-09T21:17:15,653 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#B#compaction#309 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:15,654 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/df52354832fd4baebbc573d45b4a301a is 50, key is test_row_0/B:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:15,655 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:15,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742185_1361 (size=12104) 2024-11-09T21:17:15,663 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411090f7bfb9019e64247a519cfc0e108a52d_010ea70b10f616a09e8971f27ee90066 store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:15,666 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411090f7bfb9019e64247a519cfc0e108a52d_010ea70b10f616a09e8971f27ee90066, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:15,666 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090f7bfb9019e64247a519cfc0e108a52d_010ea70b10f616a09e8971f27ee90066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:15,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742186_1362 (size=4469) 2024-11-09T21:17:15,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:15,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-09T21:17:15,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:15,790 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:15,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:15,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110948db5144ec874f0d881d5a98d5c88946_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187034393/Put/seqid=0 2024-11-09T21:17:15,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742187_1363 (size=12154) 2024-11-09T21:17:16,072 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/df52354832fd4baebbc573d45b4a301a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/df52354832fd4baebbc573d45b4a301a 2024-11-09T21:17:16,073 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:16,074 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/a42c455421334d16b4e4829933eaf695 is 175, key is test_row_0/A:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:16,079 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/B of 010ea70b10f616a09e8971f27ee90066 into df52354832fd4baebbc573d45b4a301a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:16,079 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:16,079 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/B, priority=13, startTime=1731187035642; duration=0sec 2024-11-09T21:17:16,079 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:16,079 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:B 2024-11-09T21:17:16,080 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:16,080 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:16,081 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/C is initiating minor compaction (all files) 2024-11-09T21:17:16,081 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/C in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:16,081 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/298f4da6891e4ce58be3b32b6fb70d35, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6efb403b53534b9cb24bd0041aeab76b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/e1337d484f8c487f9b5a7f87a1999039] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=35.2 K 2024-11-09T21:17:16,081 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 298f4da6891e4ce58be3b32b6fb70d35, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731187032415 2024-11-09T21:17:16,081 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 6efb403b53534b9cb24bd0041aeab76b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731187032531 2024-11-09T21:17:16,082 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e1337d484f8c487f9b5a7f87a1999039, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731187034360 2024-11-09T21:17:16,092 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#C#compaction#312 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:16,092 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/df48806a4ff645038f539324e9cf3e44 is 50, key is test_row_0/C:col10/1731187034362/Put/seqid=0 2024-11-09T21:17:16,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742188_1364 (size=31058) 2024-11-09T21:17:16,101 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/a42c455421334d16b4e4829933eaf695 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a42c455421334d16b4e4829933eaf695 2024-11-09T21:17:16,105 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/A of 010ea70b10f616a09e8971f27ee90066 into a42c455421334d16b4e4829933eaf695(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:16,105 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:16,105 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/A, priority=13, startTime=1731187035641; duration=0sec 2024-11-09T21:17:16,105 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:16,105 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:A 2024-11-09T21:17:16,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742189_1365 (size=12104) 2024-11-09T21:17:16,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:16,213 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110948db5144ec874f0d881d5a98d5c88946_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110948db5144ec874f0d881d5a98d5c88946_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:16,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c298654d0261441c8624613e51fdf0e7, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:16,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c298654d0261441c8624613e51fdf0e7 is 175, key is test_row_0/A:col10/1731187034393/Put/seqid=0 2024-11-09T21:17:16,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742190_1366 (size=30955) 2024-11-09T21:17:16,238 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c298654d0261441c8624613e51fdf0e7 2024-11-09T21:17:16,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/d0915dbb1b8a4adea5e2af0de871d854 is 50, key is test_row_0/B:col10/1731187034393/Put/seqid=0 2024-11-09T21:17:16,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742191_1367 (size=12001) 2024-11-09T21:17:16,543 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/df48806a4ff645038f539324e9cf3e44 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/df48806a4ff645038f539324e9cf3e44 2024-11-09T21:17:16,549 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/C of 010ea70b10f616a09e8971f27ee90066 into df48806a4ff645038f539324e9cf3e44(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:16,549 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:16,549 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/C, priority=13, startTime=1731187035642; duration=0sec 2024-11-09T21:17:16,550 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:16,550 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:C 2024-11-09T21:17:16,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:16,580 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:16,654 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/d0915dbb1b8a4adea5e2af0de871d854 2024-11-09T21:17:16,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187096642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187096645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187096645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,659 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187096646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/73c2c21a33eb4f8b8151757c53131867 is 50, key is test_row_0/C:col10/1731187034393/Put/seqid=0 2024-11-09T21:17:16,666 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187096658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742192_1368 (size=12001) 2024-11-09T21:17:16,682 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/73c2c21a33eb4f8b8151757c53131867 2024-11-09T21:17:16,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c298654d0261441c8624613e51fdf0e7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7 2024-11-09T21:17:16,699 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7, entries=150, sequenceid=83, filesize=30.2 K 2024-11-09T21:17:16,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/d0915dbb1b8a4adea5e2af0de871d854 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/d0915dbb1b8a4adea5e2af0de871d854 2024-11-09T21:17:16,708 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/d0915dbb1b8a4adea5e2af0de871d854, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T21:17:16,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/73c2c21a33eb4f8b8151757c53131867 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/73c2c21a33eb4f8b8151757c53131867 2024-11-09T21:17:16,715 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/73c2c21a33eb4f8b8151757c53131867, entries=150, sequenceid=83, filesize=11.7 K 2024-11-09T21:17:16,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:16,716 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 010ea70b10f616a09e8971f27ee90066 in 926ms, sequenceid=83, compaction requested=false 2024-11-09T21:17:16,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:16,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-09T21:17:16,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-09T21:17:16,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-09T21:17:16,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1490 sec 2024-11-09T21:17:16,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.1540 sec 2024-11-09T21:17:16,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:16,772 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T21:17:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:16,774 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:16,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a746df054a284e71977d8e6de0211322_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:16,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742193_1369 (size=14594) 2024-11-09T21:17:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187096801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187096802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187096814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187096817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187096817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187096926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187096928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,940 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187096937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187096937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:16,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187096937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:16,948 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187097147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187097147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,160 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187097156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187097156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187097156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,191 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:17,195 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a746df054a284e71977d8e6de0211322_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a746df054a284e71977d8e6de0211322_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:17,196 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/ddaa68020ca8476abcebdfb41dc51acc, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:17,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/ddaa68020ca8476abcebdfb41dc51acc is 175, key is test_row_0/A:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:17,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742194_1370 (size=39549) 2024-11-09T21:17:17,215 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=103, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/ddaa68020ca8476abcebdfb41dc51acc 2024-11-09T21:17:17,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/6d4414a047e44efca3fdad99205c77e4 is 50, key is test_row_0/B:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:17,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742195_1371 (size=12001) 2024-11-09T21:17:17,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187097466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187097467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,479 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187097477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187097477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,491 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:17,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187097477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:17,492 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:17,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/6d4414a047e44efca3fdad99205c77e4 2024-11-09T21:17:17,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/4b0400f012a24864af923f567a02b931 is 50, key is test_row_0/C:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:17,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742196_1372 (size=12001) 2024-11-09T21:17:17,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/4b0400f012a24864af923f567a02b931 2024-11-09T21:17:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/ddaa68020ca8476abcebdfb41dc51acc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc 2024-11-09T21:17:17,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc, entries=200, sequenceid=103, filesize=38.6 K 2024-11-09T21:17:17,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/6d4414a047e44efca3fdad99205c77e4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/6d4414a047e44efca3fdad99205c77e4 2024-11-09T21:17:17,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/6d4414a047e44efca3fdad99205c77e4, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T21:17:17,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/4b0400f012a24864af923f567a02b931 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/4b0400f012a24864af923f567a02b931 2024-11-09T21:17:17,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/4b0400f012a24864af923f567a02b931, entries=150, sequenceid=103, filesize=11.7 K 2024-11-09T21:17:17,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 010ea70b10f616a09e8971f27ee90066 in 927ms, sequenceid=103, compaction requested=true 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:17,699 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:17,699 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:17,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:17,701 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:17,701 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/B is initiating minor compaction (all files) 2024-11-09T21:17:17,701 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/B in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:17,701 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/df52354832fd4baebbc573d45b4a301a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/d0915dbb1b8a4adea5e2af0de871d854, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/6d4414a047e44efca3fdad99205c77e4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=35.3 K 2024-11-09T21:17:17,701 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:17,701 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/A is initiating minor compaction (all files) 2024-11-09T21:17:17,701 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/A in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:17,701 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a42c455421334d16b4e4829933eaf695, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=99.2 K 2024-11-09T21:17:17,701 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:17,701 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a42c455421334d16b4e4829933eaf695, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc] 2024-11-09T21:17:17,702 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting df52354832fd4baebbc573d45b4a301a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731187034360 2024-11-09T21:17:17,702 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a42c455421334d16b4e4829933eaf695, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731187034360 2024-11-09T21:17:17,702 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting d0915dbb1b8a4adea5e2af0de871d854, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731187034393 2024-11-09T21:17:17,703 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c298654d0261441c8624613e51fdf0e7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731187034393 2024-11-09T21:17:17,703 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d4414a047e44efca3fdad99205c77e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187036603 2024-11-09T21:17:17,703 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddaa68020ca8476abcebdfb41dc51acc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187036603 2024-11-09T21:17:17,725 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:17,734 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#B#compaction#319 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:17,734 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e4046a44d55143998e1e1fc6128b2336 is 50, key is test_row_0/B:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:17,749 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110907c80a656cdc4448936539cb912ffb07_010ea70b10f616a09e8971f27ee90066 store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:17,752 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110907c80a656cdc4448936539cb912ffb07_010ea70b10f616a09e8971f27ee90066, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:17,752 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110907c80a656cdc4448936539cb912ffb07_010ea70b10f616a09e8971f27ee90066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:17,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742197_1373 (size=12207) 2024-11-09T21:17:17,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742198_1374 (size=4469) 2024-11-09T21:17:17,811 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#A#compaction#318 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:17,811 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/2e1c68f651084caf8880a6e1b2c0a058 is 175, key is test_row_0/A:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:17,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742199_1375 (size=31161) 2024-11-09T21:17:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:17,990 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:17,991 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:17,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090d4b2721c99541788527a8920650d728_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187036795/Put/seqid=0 2024-11-09T21:17:18,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187098026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187098027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742200_1376 (size=14594) 2024-11-09T21:17:18,045 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:18,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187098029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,048 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,049 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090d4b2721c99541788527a8920650d728_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d4b2721c99541788527a8920650d728_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:18,051 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/712089d9c19b4e35b9c00a3b44639b0a, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:18,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/712089d9c19b4e35b9c00a3b44639b0a is 175, key is test_row_0/A:col10/1731187036795/Put/seqid=0 2024-11-09T21:17:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187098033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187098033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742201_1377 (size=39549) 2024-11-09T21:17:18,057 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/712089d9c19b4e35b9c00a3b44639b0a 2024-11-09T21:17:18,076 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/17170bf20934457681f8e50f5eb94fd8 is 50, key is test_row_0/B:col10/1731187036795/Put/seqid=0 2024-11-09T21:17:18,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742202_1378 (size=12001) 2024-11-09T21:17:18,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187098136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187098138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,142 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187098157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187098157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187098157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,173 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,201 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e4046a44d55143998e1e1fc6128b2336 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e4046a44d55143998e1e1fc6128b2336 2024-11-09T21:17:18,207 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/B of 010ea70b10f616a09e8971f27ee90066 into e4046a44d55143998e1e1fc6128b2336(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:18,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:18,207 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/B, priority=13, startTime=1731187037699; duration=0sec 2024-11-09T21:17:18,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:18,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:B 2024-11-09T21:17:18,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:18,213 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:18,213 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/C is initiating minor compaction (all files) 2024-11-09T21:17:18,213 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/C in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:18,213 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/df48806a4ff645038f539324e9cf3e44, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/73c2c21a33eb4f8b8151757c53131867, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/4b0400f012a24864af923f567a02b931] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=35.3 K 2024-11-09T21:17:18,214 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting df48806a4ff645038f539324e9cf3e44, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=66, earliestPutTs=1731187034360 2024-11-09T21:17:18,215 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 73c2c21a33eb4f8b8151757c53131867, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731187034393 2024-11-09T21:17:18,215 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b0400f012a24864af923f567a02b931, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187036603 2024-11-09T21:17:18,238 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#C#compaction#322 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:18,239 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/04a14186c5f64e8c952bc673b7cb69c8 is 50, key is test_row_0/C:col10/1731187036772/Put/seqid=0 2024-11-09T21:17:18,251 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/2e1c68f651084caf8880a6e1b2c0a058 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/2e1c68f651084caf8880a6e1b2c0a058 2024-11-09T21:17:18,259 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/A of 010ea70b10f616a09e8971f27ee90066 into 2e1c68f651084caf8880a6e1b2c0a058(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:18,259 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:18,259 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/A, priority=13, startTime=1731187037699; duration=0sec 2024-11-09T21:17:18,259 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:18,259 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:A 2024-11-09T21:17:18,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742203_1379 (size=12207) 2024-11-09T21:17:18,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187098346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,354 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187098356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187098376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187098376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,383 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187098376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,384 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/17170bf20934457681f8e50f5eb94fd8 2024-11-09T21:17:18,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/fc5dfcbe9f0e45879ca6ae0f385b92ae is 50, key is test_row_0/C:col10/1731187036795/Put/seqid=0 2024-11-09T21:17:18,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742204_1380 (size=12001) 2024-11-09T21:17:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187098656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187098676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,689 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/04a14186c5f64e8c952bc673b7cb69c8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/04a14186c5f64e8c952bc673b7cb69c8 2024-11-09T21:17:18,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187098687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187098687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,696 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/C of 010ea70b10f616a09e8971f27ee90066 into 04a14186c5f64e8c952bc673b7cb69c8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:18,696 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:18,696 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/C, priority=13, startTime=1731187037699; duration=0sec 2024-11-09T21:17:18,696 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:18,696 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:C 2024-11-09T21:17:18,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:18,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187098696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:18,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-09T21:17:18,737 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:18,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:18,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-09T21:17:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T21:17:18,756 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:18,757 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:18,757 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T21:17:18,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T21:17:18,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:18,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:18,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:18,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:18,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:18,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:18,917 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/fc5dfcbe9f0e45879ca6ae0f385b92ae 2024-11-09T21:17:18,922 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/712089d9c19b4e35b9c00a3b44639b0a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a 2024-11-09T21:17:18,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a, entries=200, sequenceid=120, filesize=38.6 K 2024-11-09T21:17:18,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/17170bf20934457681f8e50f5eb94fd8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/17170bf20934457681f8e50f5eb94fd8 2024-11-09T21:17:18,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/17170bf20934457681f8e50f5eb94fd8, entries=150, sequenceid=120, filesize=11.7 K 2024-11-09T21:17:18,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/fc5dfcbe9f0e45879ca6ae0f385b92ae as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/fc5dfcbe9f0e45879ca6ae0f385b92ae 2024-11-09T21:17:18,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/fc5dfcbe9f0e45879ca6ae0f385b92ae, entries=150, sequenceid=120, filesize=11.7 K 2024-11-09T21:17:18,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 010ea70b10f616a09e8971f27ee90066 in 950ms, sequenceid=120, compaction requested=false 2024-11-09T21:17:18,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:19,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T21:17:19,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-09T21:17:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:19,066 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:19,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a61743720b984aabb154cf27a790ce08_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187038029/Put/seqid=0 2024-11-09T21:17:19,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742205_1381 (size=12254) 2024-11-09T21:17:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:19,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187099220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187099221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,228 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187099222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187099221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187099227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187099336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187099336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187099337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,344 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187099337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187099337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,345 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T21:17:19,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:19,485 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a61743720b984aabb154cf27a790ce08_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a61743720b984aabb154cf27a790ce08_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:19,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/52d6761d841647fcb80bb7220bdbdac8, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:19,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/52d6761d841647fcb80bb7220bdbdac8 is 175, key is test_row_0/A:col10/1731187038029/Put/seqid=0 2024-11-09T21:17:19,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742206_1382 (size=31055) 2024-11-09T21:17:19,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187099557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T21:17:19,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187099877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,878 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187099877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187099880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187099880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:19,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187099881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:19,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:19,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=138, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/52d6761d841647fcb80bb7220bdbdac8 2024-11-09T21:17:19,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/aae4b1c7af60403392913b95212f3874 is 50, key is test_row_0/B:col10/1731187038029/Put/seqid=0 2024-11-09T21:17:19,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742207_1383 (size=12101) 2024-11-09T21:17:19,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/aae4b1c7af60403392913b95212f3874 2024-11-09T21:17:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/7adaa074e22c42f59ab026a3ade5075e is 50, key is test_row_0/C:col10/1731187038029/Put/seqid=0 2024-11-09T21:17:19,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742208_1384 (size=12101) 2024-11-09T21:17:19,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=138 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/7adaa074e22c42f59ab026a3ade5075e 2024-11-09T21:17:19,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/52d6761d841647fcb80bb7220bdbdac8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8 2024-11-09T21:17:19,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8, entries=150, sequenceid=138, filesize=30.3 K 2024-11-09T21:17:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/aae4b1c7af60403392913b95212f3874 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/aae4b1c7af60403392913b95212f3874 2024-11-09T21:17:19,986 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/aae4b1c7af60403392913b95212f3874, entries=150, sequenceid=138, filesize=11.8 K 2024-11-09T21:17:19,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/7adaa074e22c42f59ab026a3ade5075e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7adaa074e22c42f59ab026a3ade5075e 2024-11-09T21:17:19,992 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7adaa074e22c42f59ab026a3ade5075e, entries=150, sequenceid=138, filesize=11.8 K 2024-11-09T21:17:19,993 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 010ea70b10f616a09e8971f27ee90066 in 928ms, sequenceid=138, compaction requested=true 2024-11-09T21:17:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:19,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-09T21:17:19,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-09T21:17:19,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-09T21:17:19,997 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2380 sec 2024-11-09T21:17:19,998 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.2600 sec 2024-11-09T21:17:20,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:20,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:17:20,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:20,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:20,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:20,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:20,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:20,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:20,401 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110947adf340ed2c4b588932c204bf28f4d6_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:20,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742209_1385 (size=14794) 2024-11-09T21:17:20,417 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:20,420 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110947adf340ed2c4b588932c204bf28f4d6_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110947adf340ed2c4b588932c204bf28f4d6_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:20,421 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/6d75a10f08434e9fbd7aff3183aef649, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:20,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/6d75a10f08434e9fbd7aff3183aef649 is 175, key is test_row_0/A:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:20,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187100414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,432 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187100416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,432 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,433 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742210_1386 (size=39749) 2024-11-09T21:17:20,442 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=156, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/6d75a10f08434e9fbd7aff3183aef649 2024-11-09T21:17:20,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187100432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187100433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187100432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/9d3c62affd26460682ce311d622f68e2 is 50, key is test_row_0/B:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:20,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742211_1387 (size=12151) 2024-11-09T21:17:20,511 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/9d3c62affd26460682ce311d622f68e2 2024-11-09T21:17:20,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/681ce7c74a4c4047b9d69ceba3d52c62 is 50, key is test_row_0/C:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:20,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742212_1388 (size=12151) 2024-11-09T21:17:20,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187100537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,544 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187100537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,545 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187100557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187100557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187100557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,568 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187100757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187100757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,767 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187100776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187100776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:20,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187100776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:20,785 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-09T21:17:20,886 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:20,886 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:20,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-09T21:17:20,888 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:20,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T21:17:20,889 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:20,889 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:20,948 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/681ce7c74a4c4047b9d69ceba3d52c62 2024-11-09T21:17:20,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/6d75a10f08434e9fbd7aff3183aef649 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649 2024-11-09T21:17:20,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649, entries=200, sequenceid=156, filesize=38.8 K 2024-11-09T21:17:20,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/9d3c62affd26460682ce311d622f68e2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9d3c62affd26460682ce311d622f68e2 2024-11-09T21:17:20,962 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9d3c62affd26460682ce311d622f68e2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T21:17:20,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/681ce7c74a4c4047b9d69ceba3d52c62 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/681ce7c74a4c4047b9d69ceba3d52c62 2024-11-09T21:17:20,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/681ce7c74a4c4047b9d69ceba3d52c62, entries=150, sequenceid=156, filesize=11.9 K 2024-11-09T21:17:20,972 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 010ea70b10f616a09e8971f27ee90066 in 579ms, sequenceid=156, compaction requested=true 2024-11-09T21:17:20,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:20,972 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:17:20,973 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141514 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:17:20,973 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/A is initiating minor compaction (all files) 2024-11-09T21:17:20,973 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/A in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:20,974 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/2e1c68f651084caf8880a6e1b2c0a058, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=138.2 K 2024-11-09T21:17:20,974 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:20,974 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/2e1c68f651084caf8880a6e1b2c0a058, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649] 2024-11-09T21:17:20,974 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e1c68f651084caf8880a6e1b2c0a058, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187036603 2024-11-09T21:17:20,974 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 712089d9c19b4e35b9c00a3b44639b0a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731187036795 2024-11-09T21:17:20,975 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52d6761d841647fcb80bb7220bdbdac8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731187038010 2024-11-09T21:17:20,975 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d75a10f08434e9fbd7aff3183aef649, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731187039219 2024-11-09T21:17:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:20,980 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:17:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:20,980 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:20,982 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:20,983 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:17:20,983 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/B is initiating minor compaction (all files) 2024-11-09T21:17:20,983 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/B in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:20,983 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e4046a44d55143998e1e1fc6128b2336, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/17170bf20934457681f8e50f5eb94fd8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/aae4b1c7af60403392913b95212f3874, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9d3c62affd26460682ce311d622f68e2] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=47.3 K 2024-11-09T21:17:20,984 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e4046a44d55143998e1e1fc6128b2336, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187036603 2024-11-09T21:17:20,989 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ec8cfefb9d324f619dac204f2447f67f_010ea70b10f616a09e8971f27ee90066 store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:20,991 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ec8cfefb9d324f619dac204f2447f67f_010ea70b10f616a09e8971f27ee90066, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:20,992 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ec8cfefb9d324f619dac204f2447f67f_010ea70b10f616a09e8971f27ee90066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:20,992 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 17170bf20934457681f8e50f5eb94fd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731187036795 2024-11-09T21:17:20,993 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting aae4b1c7af60403392913b95212f3874, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731187038010 2024-11-09T21:17:20,993 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d3c62affd26460682ce311d622f68e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731187039219 2024-11-09T21:17:20,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T21:17:21,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742213_1389 (size=4469) 2024-11-09T21:17:21,013 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#A#compaction#330 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:21,014 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/cb3b72d6c7164ee9b4ccae321adc07c0 is 175, key is test_row_0/A:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:21,015 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#B#compaction#331 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:21,015 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/61f999914ac9405a927cbc15bf36e160 is 50, key is test_row_0/B:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742215_1391 (size=12493) 2024-11-09T21:17:21,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742214_1390 (size=31447) 2024-11-09T21:17:21,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:21,041 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:21,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:21,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093197caf837ac4abdb4cf839dfa02b118_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187040429/Put/seqid=0 2024-11-09T21:17:21,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742216_1392 (size=12304) 2024-11-09T21:17:21,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:21,081 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:21,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187101121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187101122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,140 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,144 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,145 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187101129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,146 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187101131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187101140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T21:17:21,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187101246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187101256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187101256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187101256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,266 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187101267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,444 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/cb3b72d6c7164ee9b4ccae321adc07c0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/cb3b72d6c7164ee9b4ccae321adc07c0 2024-11-09T21:17:21,444 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/61f999914ac9405a927cbc15bf36e160 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/61f999914ac9405a927cbc15bf36e160 2024-11-09T21:17:21,452 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/A of 010ea70b10f616a09e8971f27ee90066 into cb3b72d6c7164ee9b4ccae321adc07c0(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:21,452 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:21,452 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/A, priority=12, startTime=1731187040972; duration=0sec 2024-11-09T21:17:21,452 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:21,452 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:A 2024-11-09T21:17:21,452 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:17:21,454 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:17:21,454 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/C is initiating minor compaction (all files) 2024-11-09T21:17:21,454 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/C in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:21,454 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/04a14186c5f64e8c952bc673b7cb69c8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/fc5dfcbe9f0e45879ca6ae0f385b92ae, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7adaa074e22c42f59ab026a3ade5075e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/681ce7c74a4c4047b9d69ceba3d52c62] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=47.3 K 2024-11-09T21:17:21,461 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04a14186c5f64e8c952bc673b7cb69c8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731187036603 2024-11-09T21:17:21,461 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting fc5dfcbe9f0e45879ca6ae0f385b92ae, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731187036795 2024-11-09T21:17:21,461 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7adaa074e22c42f59ab026a3ade5075e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=138, earliestPutTs=1731187038010 2024-11-09T21:17:21,462 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 681ce7c74a4c4047b9d69ceba3d52c62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731187039219 2024-11-09T21:17:21,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:21,469 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/B of 010ea70b10f616a09e8971f27ee90066 into 61f999914ac9405a927cbc15bf36e160(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:21,469 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:21,469 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/B, priority=12, startTime=1731187040980; duration=0sec 2024-11-09T21:17:21,469 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:21,469 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:B 2024-11-09T21:17:21,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187101456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,472 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,473 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093197caf837ac4abdb4cf839dfa02b118_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093197caf837ac4abdb4cf839dfa02b118_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:21,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/a599f2dc9c1f4ea1a0a5f0dba10ab825, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:21,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/a599f2dc9c1f4ea1a0a5f0dba10ab825 is 175, key is test_row_0/A:col10/1731187040429/Put/seqid=0 2024-11-09T21:17:21,487 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#C#compaction#333 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:21,488 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/c0038b1e0bbe48daa5c99c3befe3f3e9 is 50, key is test_row_0/C:col10/1731187039219/Put/seqid=0 2024-11-09T21:17:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187101477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187101477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187101477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,490 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187101486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,495 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T21:17:21,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742217_1393 (size=31105) 2024-11-09T21:17:21,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742218_1394 (size=12493) 2024-11-09T21:17:21,547 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/c0038b1e0bbe48daa5c99c3befe3f3e9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/c0038b1e0bbe48daa5c99c3befe3f3e9 2024-11-09T21:17:21,552 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/C of 010ea70b10f616a09e8971f27ee90066 into c0038b1e0bbe48daa5c99c3befe3f3e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:21,552 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:21,553 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/C, priority=12, startTime=1731187040980; duration=0sec 2024-11-09T21:17:21,553 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:21,553 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:C 2024-11-09T21:17:21,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187101777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,783 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187101796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187101796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,808 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187101796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,811 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:21,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187101806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:21,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:21,935 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=171, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/a599f2dc9c1f4ea1a0a5f0dba10ab825 2024-11-09T21:17:21,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/baebb205995d446f97af496c37ef5b85 is 50, key is test_row_0/B:col10/1731187040429/Put/seqid=0 2024-11-09T21:17:21,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742219_1395 (size=12151) 2024-11-09T21:17:21,963 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/baebb205995d446f97af496c37ef5b85 2024-11-09T21:17:21,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/7b4c08ccf5994dc58ea5557d62622627 is 50, key is test_row_0/C:col10/1731187040429/Put/seqid=0 2024-11-09T21:17:21,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742220_1396 (size=12151) 2024-11-09T21:17:22,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T21:17:22,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187102296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,302 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187102316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:22,323 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187102317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187102317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,324 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:22,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187102317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:22,324 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:22,385 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/7b4c08ccf5994dc58ea5557d62622627 2024-11-09T21:17:22,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/a599f2dc9c1f4ea1a0a5f0dba10ab825 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825 2024-11-09T21:17:22,394 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825, entries=150, sequenceid=171, filesize=30.4 K 2024-11-09T21:17:22,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/baebb205995d446f97af496c37ef5b85 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/baebb205995d446f97af496c37ef5b85 2024-11-09T21:17:22,398 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/baebb205995d446f97af496c37ef5b85, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T21:17:22,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/7b4c08ccf5994dc58ea5557d62622627 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7b4c08ccf5994dc58ea5557d62622627 2024-11-09T21:17:22,408 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7b4c08ccf5994dc58ea5557d62622627, entries=150, sequenceid=171, filesize=11.9 K 2024-11-09T21:17:22,409 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 010ea70b10f616a09e8971f27ee90066 in 1368ms, sequenceid=171, compaction requested=false 2024-11-09T21:17:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:22,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-09T21:17:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-09T21:17:22,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-09T21:17:22,414 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5210 sec 2024-11-09T21:17:22,415 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.5280 sec 2024-11-09T21:17:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-09T21:17:23,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:23,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:23,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-09T21:17:23,039 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:23,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T21:17:23,040 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:23,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T21:17:23,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:23,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:23,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:23,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109de9742023765497faf092682463bd23b_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187041138/Put/seqid=0 2024-11-09T21:17:23,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742221_1397 (size=12304) 2024-11-09T21:17:23,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:23,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T21:17:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187103345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187103344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187103346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187103350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,357 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187103356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,469 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187103467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:23,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109de9742023765497faf092682463bd23b_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109de9742023765497faf092682463bd23b_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:23,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c0d23f4f592443ae87e39a4e5332cef2, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:23,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c0d23f4f592443ae87e39a4e5332cef2 is 175, key is test_row_0/A:col10/1731187041138/Put/seqid=0 2024-11-09T21:17:23,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742222_1398 (size=31105) 2024-11-09T21:17:23,613 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c0d23f4f592443ae87e39a4e5332cef2 2024-11-09T21:17:23,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/fed5b99cafce494bb0c47228af382762 is 50, key is test_row_0/B:col10/1731187041138/Put/seqid=0 2024-11-09T21:17:23,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742223_1399 (size=12151) 2024-11-09T21:17:23,636 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/fed5b99cafce494bb0c47228af382762 2024-11-09T21:17:23,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/721aa5bca36f4b5dab088d769b9ccc34 is 50, key is test_row_0/C:col10/1731187041138/Put/seqid=0 2024-11-09T21:17:23,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742224_1400 (size=12151) 2024-11-09T21:17:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T21:17:23,686 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187103676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187103676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187103676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,687 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187103676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187103676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187103996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187103996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187103996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187103997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187103997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,054 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/721aa5bca36f4b5dab088d769b9ccc34 2024-11-09T21:17:24,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/c0d23f4f592443ae87e39a4e5332cef2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2 2024-11-09T21:17:24,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2, entries=150, sequenceid=191, filesize=30.4 K 2024-11-09T21:17:24,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/fed5b99cafce494bb0c47228af382762 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/fed5b99cafce494bb0c47228af382762 2024-11-09T21:17:24,070 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/fed5b99cafce494bb0c47228af382762, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T21:17:24,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/721aa5bca36f4b5dab088d769b9ccc34 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/721aa5bca36f4b5dab088d769b9ccc34 2024-11-09T21:17:24,074 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/721aa5bca36f4b5dab088d769b9ccc34, entries=150, sequenceid=191, filesize=11.9 K 2024-11-09T21:17:24,075 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 010ea70b10f616a09e8971f27ee90066 in 883ms, sequenceid=191, compaction requested=true 2024-11-09T21:17:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:24,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-09T21:17:24,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-09T21:17:24,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-09T21:17:24,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0360 sec 2024-11-09T21:17:24,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.0410 sec 2024-11-09T21:17:24,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-09T21:17:24,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:24,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:24,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-09T21:17:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T21:17:24,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:24,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:24,179 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:24,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T21:17:24,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-09T21:17:24,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:24,331 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:24,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:24,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090ea55c94275a4d46a34c4618c0e3f631_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187043345/Put/seqid=0 2024-11-09T21:17:24,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742225_1401 (size=12304) 2024-11-09T21:17:24,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:24,357 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411090ea55c94275a4d46a34c4618c0e3f631_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090ea55c94275a4d46a34c4618c0e3f631_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/050c55f5d8744f16a5df7b2ea77a43eb, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:24,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/050c55f5d8744f16a5df7b2ea77a43eb is 175, key is test_row_0/A:col10/1731187043345/Put/seqid=0 2024-11-09T21:17:24,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742226_1402 (size=31105) 2024-11-09T21:17:24,396 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/050c55f5d8744f16a5df7b2ea77a43eb 2024-11-09T21:17:24,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/efc72d520d4b48a3929a7318859f0f0f is 50, key is test_row_0/B:col10/1731187043345/Put/seqid=0 2024-11-09T21:17:24,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742227_1403 (size=12151) 2024-11-09T21:17:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T21:17:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:24,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187104545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187104546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187104566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187104568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,588 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187104569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,589 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187104676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187104676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,688 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187104697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187104697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187104697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T21:17:24,818 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/efc72d520d4b48a3929a7318859f0f0f 2024-11-09T21:17:24,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/ee4d8dd6d1cb40d4bac2d64162487f09 is 50, key is test_row_0/C:col10/1731187043345/Put/seqid=0 2024-11-09T21:17:24,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742228_1404 (size=12151) 2024-11-09T21:17:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187104896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,899 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187104897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187104916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187104916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:24,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:24,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187104916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:24,931 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187105206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187105206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,216 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,217 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,237 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/ee4d8dd6d1cb40d4bac2d64162487f09 2024-11-09T21:17:25,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/050c55f5d8744f16a5df7b2ea77a43eb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb 2024-11-09T21:17:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187105237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187105237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187105237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,243 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,245 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb, entries=150, sequenceid=206, filesize=30.4 K 2024-11-09T21:17:25,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/efc72d520d4b48a3929a7318859f0f0f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/efc72d520d4b48a3929a7318859f0f0f 2024-11-09T21:17:25,249 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/efc72d520d4b48a3929a7318859f0f0f, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T21:17:25,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/ee4d8dd6d1cb40d4bac2d64162487f09 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ee4d8dd6d1cb40d4bac2d64162487f09 2024-11-09T21:17:25,253 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ee4d8dd6d1cb40d4bac2d64162487f09, entries=150, sequenceid=206, filesize=11.9 K 2024-11-09T21:17:25,253 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 010ea70b10f616a09e8971f27ee90066 in 922ms, sequenceid=206, compaction requested=true 2024-11-09T21:17:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:25,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-09T21:17:25,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-09T21:17:25,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-09T21:17:25,256 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0750 sec 2024-11-09T21:17:25,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.0790 sec 2024-11-09T21:17:25,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-09T21:17:25,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:25,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:25,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-09T21:17:25,319 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T21:17:25,320 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:25,320 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T21:17:25,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-09T21:17:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:25,473 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:25,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110910141a551e08419da08f878192b5e89d_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187044565/Put/seqid=0 2024-11-09T21:17:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742229_1405 (size=12304) 2024-11-09T21:17:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T21:17:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:25,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:25,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187105755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,767 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187105756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187105757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,775 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187105758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187105767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:25,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187105876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,892 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110910141a551e08419da08f878192b5e89d_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110910141a551e08419da08f878192b5e89d_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:25,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f51311f035e24d4dbd7b7983cca6ae20, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:25,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f51311f035e24d4dbd7b7983cca6ae20 is 175, key is test_row_0/A:col10/1731187044565/Put/seqid=0 2024-11-09T21:17:25,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742230_1406 (size=31105) 2024-11-09T21:17:25,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187105886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187105886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187105886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:25,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T21:17:26,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187106096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,106 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187106106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187106107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187106107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,299 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=223, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f51311f035e24d4dbd7b7983cca6ae20 2024-11-09T21:17:26,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e3fffde625a14a57b360bf54b71a8754 is 50, key is test_row_0/B:col10/1731187044565/Put/seqid=0 2024-11-09T21:17:26,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742231_1407 (size=12151) 2024-11-09T21:17:26,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187106416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187106416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187106416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187106416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,426 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T21:17:26,711 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e3fffde625a14a57b360bf54b71a8754 2024-11-09T21:17:26,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/60ab3481cb9d4579bb9b4e0d4f0eac1d is 50, key is test_row_0/C:col10/1731187044565/Put/seqid=0 2024-11-09T21:17:26,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742232_1408 (size=12151) 2024-11-09T21:17:26,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187106787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,798 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in 010ea70b10f616a09e8971f27ee90066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,798 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187106936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187106936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187106936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187106936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:27,120 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/60ab3481cb9d4579bb9b4e0d4f0eac1d 2024-11-09T21:17:27,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/f51311f035e24d4dbd7b7983cca6ae20 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20 2024-11-09T21:17:27,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20, entries=150, sequenceid=223, filesize=30.4 K 2024-11-09T21:17:27,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e3fffde625a14a57b360bf54b71a8754 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e3fffde625a14a57b360bf54b71a8754 2024-11-09T21:17:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,130 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e3fffde625a14a57b360bf54b71a8754, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T21:17:27,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/60ab3481cb9d4579bb9b4e0d4f0eac1d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/60ab3481cb9d4579bb9b4e0d4f0eac1d 2024-11-09T21:17:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,134 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/60ab3481cb9d4579bb9b4e0d4f0eac1d, entries=150, sequenceid=223, filesize=11.9 K 2024-11-09T21:17:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,135 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 010ea70b10f616a09e8971f27ee90066 in 1663ms, sequenceid=223, compaction requested=true 2024-11-09T21:17:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:27,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-09T21:17:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-09T21:17:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-09T21:17:27,139 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8160 sec 2024-11-09T21:17:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.8220 sec 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-09T21:17:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,477 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-09T21:17:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,479 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:27,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T21:17:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,480 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:27,480 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:27,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T21:17:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-09T21:17:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:27,635 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:27,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:27,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:27,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c8a40a6089e64b7b97703ea9307a20cc_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187045756/Put/seqid=0 2024-11-09T21:17:27,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742233_1409 (size=9814) 2024-11-09T21:17:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T21:17:27,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:27,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187108017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187108015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,042 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c8a40a6089e64b7b97703ea9307a20cc_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c8a40a6089e64b7b97703ea9307a20cc_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:28,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/69e0af43d46e4cc1beca22ef3bd9b04e, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:28,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/69e0af43d46e4cc1beca22ef3bd9b04e is 175, key is test_row_0/A:col10/1731187045756/Put/seqid=0 2024-11-09T21:17:28,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187108042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187108043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742234_1410 (size=22461) 2024-11-09T21:17:28,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=238, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/69e0af43d46e4cc1beca22ef3bd9b04e 2024-11-09T21:17:28,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T21:17:28,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e27e2ab450fc499d8c3965dcd9369609 is 50, key is test_row_0/B:col10/1731187045756/Put/seqid=0 2024-11-09T21:17:28,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742235_1411 (size=9757) 2024-11-09T21:17:28,158 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e27e2ab450fc499d8c3965dcd9369609 2024-11-09T21:17:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187108146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187108146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/ed62bef3a6984e72bafc1e52535ac917 is 50, key is test_row_0/C:col10/1731187045756/Put/seqid=0 2024-11-09T21:17:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187108176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187108176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742236_1412 (size=9757) 2024-11-09T21:17:28,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=238 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/ed62bef3a6984e72bafc1e52535ac917 2024-11-09T21:17:28,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/69e0af43d46e4cc1beca22ef3bd9b04e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e 2024-11-09T21:17:28,245 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e, entries=100, sequenceid=238, filesize=21.9 K 2024-11-09T21:17:28,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/e27e2ab450fc499d8c3965dcd9369609 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e27e2ab450fc499d8c3965dcd9369609 2024-11-09T21:17:28,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e27e2ab450fc499d8c3965dcd9369609, entries=100, sequenceid=238, filesize=9.5 K 2024-11-09T21:17:28,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/ed62bef3a6984e72bafc1e52535ac917 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ed62bef3a6984e72bafc1e52535ac917 2024-11-09T21:17:28,271 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ed62bef3a6984e72bafc1e52535ac917, entries=100, sequenceid=238, filesize=9.5 K 2024-11-09T21:17:28,272 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 010ea70b10f616a09e8971f27ee90066 in 637ms, sequenceid=238, compaction requested=true 2024-11-09T21:17:28,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:28,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:28,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-09T21:17:28,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-09T21:17:28,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-09T21:17:28,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 797 msec 2024-11-09T21:17:28,280 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 802 msec 2024-11-09T21:17:28,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:28,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:17:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:28,379 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:28,411 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099bee153832c247b6bfc9300a16b15c54_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:28,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742237_1413 (size=14794) 2024-11-09T21:17:28,426 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:28,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099bee153832c247b6bfc9300a16b15c54_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099bee153832c247b6bfc9300a16b15c54_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:28,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4a1bfcd97ec2423d847a18ba7bb33068, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:28,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4a1bfcd97ec2423d847a18ba7bb33068 is 175, key is test_row_0/A:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187108431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187108433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187108445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,468 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187108445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,469 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742238_1414 (size=39749) 2024-11-09T21:17:28,489 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4a1bfcd97ec2423d847a18ba7bb33068 2024-11-09T21:17:28,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/0839dfa7ba0842fcaf66af4594d96f43 is 50, key is test_row_0/B:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:28,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742239_1415 (size=12151) 2024-11-09T21:17:28,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/0839dfa7ba0842fcaf66af4594d96f43 2024-11-09T21:17:28,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/248701ed3fa3417a92fd9acc3cc27702 is 50, key is test_row_0/C:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:28,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187108567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,581 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187108567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,582 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187108576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187108576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742240_1416 (size=12151) 2024-11-09T21:17:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-09T21:17:28,620 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:28,620 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:28,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-09T21:17:28,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T21:17:28,625 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:28,629 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:28,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T21:17:28,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T21:17:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:28,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:28,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:28,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:28,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187108787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187108787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187108797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,804 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187108799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,807 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:28,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56246 deadline: 1731187108816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:28,824 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in 010ea70b10f616a09e8971f27ee90066 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4255 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:28,824 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:28,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T21:17:28,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T21:17:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:28,954 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:28,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:29,035 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/248701ed3fa3417a92fd9acc3cc27702 2024-11-09T21:17:29,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4a1bfcd97ec2423d847a18ba7bb33068 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068 2024-11-09T21:17:29,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068, entries=200, sequenceid=256, filesize=38.8 K 2024-11-09T21:17:29,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/0839dfa7ba0842fcaf66af4594d96f43 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/0839dfa7ba0842fcaf66af4594d96f43 2024-11-09T21:17:29,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/0839dfa7ba0842fcaf66af4594d96f43, entries=150, sequenceid=256, filesize=11.9 K 2024-11-09T21:17:29,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/248701ed3fa3417a92fd9acc3cc27702 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/248701ed3fa3417a92fd9acc3cc27702 2024-11-09T21:17:29,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/248701ed3fa3417a92fd9acc3cc27702, entries=150, sequenceid=256, filesize=11.9 K 2024-11-09T21:17:29,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 010ea70b10f616a09e8971f27ee90066 in 673ms, sequenceid=256, compaction requested=true 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:29,052 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:29,052 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:29,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:29,054 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 218077 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-09T21:17:29,054 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 83005 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/A is initiating minor compaction (all files) 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/B is initiating minor compaction (all files) 2024-11-09T21:17:29,055 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/A in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:29,055 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/B in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:29,055 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/cb3b72d6c7164ee9b4ccae321adc07c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=213.0 K 2024-11-09T21:17:29,055 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/61f999914ac9405a927cbc15bf36e160, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/baebb205995d446f97af496c37ef5b85, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/fed5b99cafce494bb0c47228af382762, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/efc72d520d4b48a3929a7318859f0f0f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e3fffde625a14a57b360bf54b71a8754, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e27e2ab450fc499d8c3965dcd9369609, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/0839dfa7ba0842fcaf66af4594d96f43] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=81.1 K 2024-11-09T21:17:29,055 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=9 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/cb3b72d6c7164ee9b4ccae321adc07c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068] 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb3b72d6c7164ee9b4ccae321adc07c0, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731187039219 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 61f999914ac9405a927cbc15bf36e160, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731187039219 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting baebb205995d446f97af496c37ef5b85, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731187040410 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a599f2dc9c1f4ea1a0a5f0dba10ab825, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731187040410 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fed5b99cafce494bb0c47228af382762, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731187041121 2024-11-09T21:17:29,055 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d23f4f592443ae87e39a4e5332cef2, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731187041121 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting efc72d520d4b48a3929a7318859f0f0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731187043345 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 050c55f5d8744f16a5df7b2ea77a43eb, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731187043345 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e3fffde625a14a57b360bf54b71a8754, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731187044540 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f51311f035e24d4dbd7b7983cca6ae20, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731187044540 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e27e2ab450fc499d8c3965dcd9369609, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731187045756 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69e0af43d46e4cc1beca22ef3bd9b04e, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731187045756 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0839dfa7ba0842fcaf66af4594d96f43, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731187048034 2024-11-09T21:17:29,056 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a1bfcd97ec2423d847a18ba7bb33068, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731187047999 2024-11-09T21:17:29,085 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#B#compaction#351 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:29,085 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a82fde50bf5c492d85280ef98e9b41eb is 50, key is test_row_0/B:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:29,086 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:29,088 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109ca32097ce10847dd838d405f1edef40d_010ea70b10f616a09e8971f27ee90066 store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:29,093 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109ca32097ce10847dd838d405f1edef40d_010ea70b10f616a09e8971f27ee90066, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:29,093 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ca32097ce10847dd838d405f1edef40d_010ea70b10f616a09e8971f27ee90066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742242_1418 (size=4469) 2024-11-09T21:17:29,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742241_1417 (size=12731) 2024-11-09T21:17:29,106 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#A#compaction#352 average throughput is 1.22 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:29,106 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4f37d2b9f6ee4d82a69f8c5960350f76 is 175, key is test_row_0/A:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:29,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-09T21:17:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:29,107 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T21:17:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:29,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:29,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742243_1419 (size=31685) 2024-11-09T21:17:29,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a5c130acac5a4dc9bca4e9a0243d56d1_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187048436/Put/seqid=0 2024-11-09T21:17:29,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742244_1420 (size=12454) 2024-11-09T21:17:29,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:29,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:29,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187109166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187109183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187109204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187109203, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T21:17:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187109306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187109307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187109336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,345 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187109336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,346 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,510 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/a82fde50bf5c492d85280ef98e9b41eb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a82fde50bf5c492d85280ef98e9b41eb 2024-11-09T21:17:29,514 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/B of 010ea70b10f616a09e8971f27ee90066 into a82fde50bf5c492d85280ef98e9b41eb(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:29,514 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:29,514 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/B, priority=9, startTime=1731187049052; duration=0sec 2024-11-09T21:17:29,514 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:29,514 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:B 2024-11-09T21:17:29,514 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 7 store files, 0 compacting, 7 eligible, 16 blocking 2024-11-09T21:17:29,516 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 7 files of size 83005 starting at candidate #0 after considering 15 permutations with 15 in ratio 2024-11-09T21:17:29,516 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/C is initiating minor compaction (all files) 2024-11-09T21:17:29,516 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/C in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:29,516 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/c0038b1e0bbe48daa5c99c3befe3f3e9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7b4c08ccf5994dc58ea5557d62622627, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/721aa5bca36f4b5dab088d769b9ccc34, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ee4d8dd6d1cb40d4bac2d64162487f09, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/60ab3481cb9d4579bb9b4e0d4f0eac1d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ed62bef3a6984e72bafc1e52535ac917, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/248701ed3fa3417a92fd9acc3cc27702] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=81.1 K 2024-11-09T21:17:29,517 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c0038b1e0bbe48daa5c99c3befe3f3e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731187039219 2024-11-09T21:17:29,517 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b4c08ccf5994dc58ea5557d62622627, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731187040410 2024-11-09T21:17:29,518 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 721aa5bca36f4b5dab088d769b9ccc34, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731187041121 2024-11-09T21:17:29,518 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ee4d8dd6d1cb40d4bac2d64162487f09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731187043345 2024-11-09T21:17:29,518 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 60ab3481cb9d4579bb9b4e0d4f0eac1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731187044540 2024-11-09T21:17:29,519 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ed62bef3a6984e72bafc1e52535ac917, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=238, earliestPutTs=1731187045756 2024-11-09T21:17:29,519 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 248701ed3fa3417a92fd9acc3cc27702, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731187048034 2024-11-09T21:17:29,522 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4f37d2b9f6ee4d82a69f8c5960350f76 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4f37d2b9f6ee4d82a69f8c5960350f76 2024-11-09T21:17:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:29,529 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/A of 010ea70b10f616a09e8971f27ee90066 into 4f37d2b9f6ee4d82a69f8c5960350f76(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:29,529 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:29,529 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/A, priority=9, startTime=1731187049052; duration=0sec 2024-11-09T21:17:29,529 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:29,529 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:A 2024-11-09T21:17:29,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a5c130acac5a4dc9bca4e9a0243d56d1_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a5c130acac5a4dc9bca4e9a0243d56d1_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4ae672a8e7ee44738524e2db3dbaff49, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:29,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4ae672a8e7ee44738524e2db3dbaff49 is 175, key is test_row_0/A:col10/1731187048436/Put/seqid=0 2024-11-09T21:17:29,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187109517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187109517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,536 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,538 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742245_1421 (size=31255) 2024-11-09T21:17:29,545 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#C#compaction#354 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:29,545 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/812717c7221c41e89ca567b34de8bea1 is 50, key is test_row_0/C:col10/1731187048377/Put/seqid=0 2024-11-09T21:17:29,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742246_1422 (size=12731) 2024-11-09T21:17:29,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187109557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,573 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187109560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,577 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T21:17:29,852 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187109847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187109847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187109866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,870 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:29,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187109886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:29,895 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:29,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4ae672a8e7ee44738524e2db3dbaff49 2024-11-09T21:17:29,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/893de30f03d0464db515736d564de1e9 is 50, key is test_row_0/B:col10/1731187048436/Put/seqid=0 2024-11-09T21:17:29,954 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/812717c7221c41e89ca567b34de8bea1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/812717c7221c41e89ca567b34de8bea1 2024-11-09T21:17:29,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742247_1423 (size=12301) 2024-11-09T21:17:29,956 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/893de30f03d0464db515736d564de1e9 2024-11-09T21:17:29,974 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 7 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/C of 010ea70b10f616a09e8971f27ee90066 into 812717c7221c41e89ca567b34de8bea1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:29,974 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:29,974 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/C, priority=9, startTime=1731187049052; duration=0sec 2024-11-09T21:17:29,974 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:29,974 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:C 2024-11-09T21:17:29,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/aa1c92a811f0434481321f86757e3040 is 50, key is test_row_0/C:col10/1731187048436/Put/seqid=0 2024-11-09T21:17:30,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742248_1424 (size=12301) 2024-11-09T21:17:30,014 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/aa1c92a811f0434481321f86757e3040 2024-11-09T21:17:30,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/4ae672a8e7ee44738524e2db3dbaff49 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49 2024-11-09T21:17:30,030 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49, entries=150, sequenceid=270, filesize=30.5 K 2024-11-09T21:17:30,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/893de30f03d0464db515736d564de1e9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/893de30f03d0464db515736d564de1e9 2024-11-09T21:17:30,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/893de30f03d0464db515736d564de1e9, entries=150, sequenceid=270, filesize=12.0 K 2024-11-09T21:17:30,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/aa1c92a811f0434481321f86757e3040 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/aa1c92a811f0434481321f86757e3040 2024-11-09T21:17:30,040 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/aa1c92a811f0434481321f86757e3040, entries=150, sequenceid=270, filesize=12.0 K 2024-11-09T21:17:30,040 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 010ea70b10f616a09e8971f27ee90066 in 933ms, sequenceid=270, compaction requested=false 2024-11-09T21:17:30,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:30,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:30,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-09T21:17:30,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-09T21:17:30,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-09T21:17:30,043 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4120 sec 2024-11-09T21:17:30,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.4230 sec 2024-11-09T21:17:30,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:30,366 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:17:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:30,366 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:30,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:30,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:30,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ff0e0ea151764621a71399ae7edd56bd_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:30,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742249_1425 (size=14994) 2024-11-09T21:17:30,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187110392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,406 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187110393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,406 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,407 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187110401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,409 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,416 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187110406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,417 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187110506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187110516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,521 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187110516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,525 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187110716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187110726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,735 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187110736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:30,743 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:30,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-09T21:17:30,766 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:30,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-09T21:17:30,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:30,768 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:30,769 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:30,769 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:30,777 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:30,780 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109ff0e0ea151764621a71399ae7edd56bd_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ff0e0ea151764621a71399ae7edd56bd_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:30,780 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/7f011b976f3b4c3199961539aee840cc, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:30,781 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/7f011b976f3b4c3199961539aee840cc is 175, key is test_row_0/A:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:30,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742250_1426 (size=39949) 2024-11-09T21:17:30,785 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/7f011b976f3b4c3199961539aee840cc 2024-11-09T21:17:30,791 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/985ca91f738844d8b3aa8d7461d953e3 is 50, key is test_row_0/B:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:30,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742251_1427 (size=12301) 2024-11-09T21:17:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:30,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T21:17:30,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:30,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:30,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:30,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:30,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:30,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187111026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187111047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187111047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T21:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:31,199 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/985ca91f738844d8b3aa8d7461d953e3 2024-11-09T21:17:31,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/03d47e8f633b4820866bbdd37d7cf32b is 50, key is test_row_0/C:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:31,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742252_1428 (size=12301) 2024-11-09T21:17:31,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T21:17:31,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T21:17:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:31,435 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56206 deadline: 1731187111426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,435 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in 010ea70b10f616a09e8971f27ee90066 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T21:17:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:31,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56218 deadline: 1731187111546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56232 deadline: 1731187111556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56242 deadline: 1731187111556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:17:31,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/03d47e8f633b4820866bbdd37d7cf32b 2024-11-09T21:17:31,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/7f011b976f3b4c3199961539aee840cc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc 2024-11-09T21:17:31,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc, entries=200, sequenceid=293, filesize=39.0 K 2024-11-09T21:17:31,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/985ca91f738844d8b3aa8d7461d953e3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/985ca91f738844d8b3aa8d7461d953e3 2024-11-09T21:17:31,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/985ca91f738844d8b3aa8d7461d953e3, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T21:17:31,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/03d47e8f633b4820866bbdd37d7cf32b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/03d47e8f633b4820866bbdd37d7cf32b 2024-11-09T21:17:31,635 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/03d47e8f633b4820866bbdd37d7cf32b, entries=150, sequenceid=293, filesize=12.0 K 2024-11-09T21:17:31,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 010ea70b10f616a09e8971f27ee90066 in 1270ms, sequenceid=293, compaction requested=true 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:31,636 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:31,636 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 010ea70b10f616a09e8971f27ee90066:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:31,636 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:31,649 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:31,649 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/A is initiating minor compaction (all files) 2024-11-09T21:17:31,649 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/A in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,649 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4f37d2b9f6ee4d82a69f8c5960350f76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=100.5 K 2024-11-09T21:17:31,649 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,649 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4f37d2b9f6ee4d82a69f8c5960350f76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc] 2024-11-09T21:17:31,649 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:31,649 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/B is initiating minor compaction (all files) 2024-11-09T21:17:31,649 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/B in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,649 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a82fde50bf5c492d85280ef98e9b41eb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/893de30f03d0464db515736d564de1e9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/985ca91f738844d8b3aa8d7461d953e3] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=36.5 K 2024-11-09T21:17:31,650 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a82fde50bf5c492d85280ef98e9b41eb, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731187048034 2024-11-09T21:17:31,650 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f37d2b9f6ee4d82a69f8c5960350f76, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731187048034 2024-11-09T21:17:31,650 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae672a8e7ee44738524e2db3dbaff49, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731187048414 2024-11-09T21:17:31,650 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 893de30f03d0464db515736d564de1e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731187048414 2024-11-09T21:17:31,651 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f011b976f3b4c3199961539aee840cc, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731187049160 2024-11-09T21:17:31,651 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 985ca91f738844d8b3aa8d7461d953e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731187049181 2024-11-09T21:17:31,666 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:31,666 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/bdaf78c35c2444319889b739831b665d is 50, key is test_row_0/B:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:31,669 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:31,679 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109064756ecdd58485c90a8dc12d79b4c16_010ea70b10f616a09e8971f27ee90066 store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:31,681 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109064756ecdd58485c90a8dc12d79b4c16_010ea70b10f616a09e8971f27ee90066, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:31,681 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109064756ecdd58485c90a8dc12d79b4c16_010ea70b10f616a09e8971f27ee90066 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:31,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-09T21:17:31,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:31,685 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:17:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:31,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:31,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742253_1429 (size=12983) 2024-11-09T21:17:31,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742254_1430 (size=4469) 2024-11-09T21:17:31,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110953af2db654344a368fa5ef2b20865145_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_0/A:col10/1731187050391/Put/seqid=0 2024-11-09T21:17:31,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742255_1431 (size=12454) 2024-11-09T21:17:31,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:31,732 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110953af2db654344a368fa5ef2b20865145_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110953af2db654344a368fa5ef2b20865145_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:31,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/29474628994d41d2883b25a4e38eb0cf, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:31,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/29474628994d41d2883b25a4e38eb0cf is 175, key is test_row_0/A:col10/1731187050391/Put/seqid=0 2024-11-09T21:17:31,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742256_1432 (size=31255) 2024-11-09T21:17:31,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:32,098 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/bdaf78c35c2444319889b739831b665d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/bdaf78c35c2444319889b739831b665d 2024-11-09T21:17:32,098 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#A#compaction#361 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:32,098 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/34ef54a398ee4914bbc4092f95a1d541 is 175, key is test_row_0/A:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:32,102 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/B of 010ea70b10f616a09e8971f27ee90066 into bdaf78c35c2444319889b739831b665d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:32,102 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:32,102 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/B, priority=13, startTime=1731187051636; duration=0sec 2024-11-09T21:17:32,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742257_1433 (size=31937) 2024-11-09T21:17:32,102 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:32,102 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:B 2024-11-09T21:17:32,102 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:32,103 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:32,103 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 010ea70b10f616a09e8971f27ee90066/C is initiating minor compaction (all files) 2024-11-09T21:17:32,103 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 010ea70b10f616a09e8971f27ee90066/C in TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:32,103 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/812717c7221c41e89ca567b34de8bea1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/aa1c92a811f0434481321f86757e3040, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/03d47e8f633b4820866bbdd37d7cf32b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp, totalSize=36.5 K 2024-11-09T21:17:32,103 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 812717c7221c41e89ca567b34de8bea1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731187048034 2024-11-09T21:17:32,104 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting aa1c92a811f0434481321f86757e3040, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731187048414 2024-11-09T21:17:32,104 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 03d47e8f633b4820866bbdd37d7cf32b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731187049181 2024-11-09T21:17:32,109 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 010ea70b10f616a09e8971f27ee90066#C#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:32,109 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/95435a01492e4412875fe1c89f2f920c is 50, key is test_row_0/C:col10/1731187050365/Put/seqid=0 2024-11-09T21:17:32,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742258_1434 (size=12983) 2024-11-09T21:17:32,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=305, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/29474628994d41d2883b25a4e38eb0cf 2024-11-09T21:17:32,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/64aafd3d32664fc8bec187e0e33a43ba is 50, key is test_row_0/B:col10/1731187050391/Put/seqid=0 2024-11-09T21:17:32,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742259_1435 (size=12301) 2024-11-09T21:17:32,418 INFO [Thread-1595 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1595. 2024-11-09T21:17:32,418 DEBUG [Thread-1595 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,418 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,418 DEBUG [Thread-1595 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,418 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,421 INFO [Thread-1601 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1601. 2024-11-09T21:17:32,421 DEBUG [Thread-1601 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,421 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,422 DEBUG [Thread-1601 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,422 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,423 INFO [Thread-1597 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1597. 2024-11-09T21:17:32,423 DEBUG [Thread-1597 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,423 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,423 DEBUG [Thread-1597 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,423 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,425 INFO [Thread-1599 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1599. 2024-11-09T21:17:32,425 DEBUG [Thread-1599 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,425 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,425 DEBUG [Thread-1599 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,425 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,428 INFO [Thread-1593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1593. 2024-11-09T21:17:32,428 DEBUG [Thread-1593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,428 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,428 DEBUG [Thread-1593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,514 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/34ef54a398ee4914bbc4092f95a1d541 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/34ef54a398ee4914bbc4092f95a1d541 2024-11-09T21:17:32,520 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/A of 010ea70b10f616a09e8971f27ee90066 into 34ef54a398ee4914bbc4092f95a1d541(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:32,520 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/95435a01492e4412875fe1c89f2f920c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/95435a01492e4412875fe1c89f2f920c 2024-11-09T21:17:32,520 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:32,520 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/A, priority=13, startTime=1731187051636; duration=0sec 2024-11-09T21:17:32,521 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:32,521 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:A 2024-11-09T21:17:32,525 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 010ea70b10f616a09e8971f27ee90066/C of 010ea70b10f616a09e8971f27ee90066 into 95435a01492e4412875fe1c89f2f920c(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:32,525 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:32,525 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., storeName=010ea70b10f616a09e8971f27ee90066/C, priority=13, startTime=1731187051636; duration=0sec 2024-11-09T21:17:32,525 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:32,525 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 010ea70b10f616a09e8971f27ee90066:C 2024-11-09T21:17:32,552 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/64aafd3d32664fc8bec187e0e33a43ba 2024-11-09T21:17:32,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/b407d45c476e4756b044563ff65ea18b is 50, key is test_row_0/C:col10/1731187050391/Put/seqid=0 2024-11-09T21:17:32,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742260_1436 (size=12301) 2024-11-09T21:17:32,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:32,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. as already flushing 2024-11-09T21:17:32,602 INFO [Thread-1584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1584. 2024-11-09T21:17:32,602 DEBUG [Thread-1584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,602 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,602 INFO [Thread-1582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1582. 2024-11-09T21:17:32,602 DEBUG [Thread-1582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,602 INFO [Thread-1588 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1588. 2024-11-09T21:17:32,602 DEBUG [Thread-1584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,602 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,602 DEBUG [Thread-1588 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,602 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,602 DEBUG [Thread-1582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,602 DEBUG [Thread-1588 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,602 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,870 INFO [Thread-1590 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1590. 2024-11-09T21:17:32,870 DEBUG [Thread-1590 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:32,871 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,871 DEBUG [Thread-1590 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:32,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:32,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:32,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=305 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/b407d45c476e4756b044563ff65ea18b 2024-11-09T21:17:32,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/29474628994d41d2883b25a4e38eb0cf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/29474628994d41d2883b25a4e38eb0cf 2024-11-09T21:17:32,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/29474628994d41d2883b25a4e38eb0cf, entries=150, sequenceid=305, filesize=30.5 K 2024-11-09T21:17:32,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/64aafd3d32664fc8bec187e0e33a43ba as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/64aafd3d32664fc8bec187e0e33a43ba 2024-11-09T21:17:32,975 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/64aafd3d32664fc8bec187e0e33a43ba, entries=150, sequenceid=305, filesize=12.0 K 2024-11-09T21:17:32,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/b407d45c476e4756b044563ff65ea18b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/b407d45c476e4756b044563ff65ea18b 2024-11-09T21:17:32,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/b407d45c476e4756b044563ff65ea18b, entries=150, sequenceid=305, filesize=12.0 K 2024-11-09T21:17:32,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=26.84 KB/27480 for 010ea70b10f616a09e8971f27ee90066 in 1294ms, sequenceid=305, compaction requested=false 2024-11-09T21:17:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 010ea70b10f616a09e8971f27ee90066: 2024-11-09T21:17:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:32,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-09T21:17:32,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-09T21:17:32,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-09T21:17:32,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2120 sec 2024-11-09T21:17:32,984 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 2.2160 sec 2024-11-09T21:17:33,223 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:17:33,458 INFO [Thread-1586 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1586. 2024-11-09T21:17:33,458 DEBUG [Thread-1586 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:33,458 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:33,458 DEBUG [Thread-1586 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:33,458 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:34,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-09T21:17:34,947 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 959 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2864 rows 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 947 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2831 rows 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 954 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2849 rows 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 950 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2840 rows 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 960 2024-11-09T21:17:34,947 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2876 rows 2024-11-09T21:17:34,947 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:17:34,947 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:17:34,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:34,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:34,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:34,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T21:17:34,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:34,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T21:17:34,951 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187054951"}]},"ts":"1731187054951"} 2024-11-09T21:17:34,952 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T21:17:34,953 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T21:17:34,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:17:34,954 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, UNASSIGN}] 2024-11-09T21:17:34,955 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, UNASSIGN 2024-11-09T21:17:34,955 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:34,957 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=121, ppid=120, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, UNASSIGN because future has completed 2024-11-09T21:17:34,957 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:17:34,957 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:17:35,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T21:17:35,110 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(122): Close 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:35,110 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:17:35,110 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1722): Closing 010ea70b10f616a09e8971f27ee90066, disabling compactions & flushes 2024-11-09T21:17:35,110 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:35,110 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:35,110 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. after waiting 0 ms 2024-11-09T21:17:35,110 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:35,110 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(2902): Flushing 010ea70b10f616a09e8971f27ee90066 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T21:17:35,111 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=A 2024-11-09T21:17:35,111 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:35,111 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=B 2024-11-09T21:17:35,111 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:35,111 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 010ea70b10f616a09e8971f27ee90066, store=C 2024-11-09T21:17:35,111 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:35,120 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a923b43f1c26415c958da2965ce7ac5a_010ea70b10f616a09e8971f27ee90066 is 50, key is test_row_2/A:col10/1731187053456/Put/seqid=0 2024-11-09T21:17:35,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742261_1437 (size=7374) 2024-11-09T21:17:35,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T21:17:35,524 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:35,530 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a923b43f1c26415c958da2965ce7ac5a_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a923b43f1c26415c958da2965ce7ac5a_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:35,532 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/347eaf6212274208a8d0521f5b480971, store: [table=TestAcidGuarantees family=A region=010ea70b10f616a09e8971f27ee90066] 2024-11-09T21:17:35,533 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/347eaf6212274208a8d0521f5b480971 is 175, key is test_row_2/A:col10/1731187053456/Put/seqid=0 2024-11-09T21:17:35,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742262_1438 (size=13865) 2024-11-09T21:17:35,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T21:17:35,938 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=316, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/347eaf6212274208a8d0521f5b480971 2024-11-09T21:17:35,943 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/75a691680bf6421aaad9e985dad4b44b is 50, key is test_row_2/B:col10/1731187053456/Put/seqid=0 2024-11-09T21:17:35,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742263_1439 (size=7415) 2024-11-09T21:17:36,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T21:17:36,346 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/75a691680bf6421aaad9e985dad4b44b 2024-11-09T21:17:36,351 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/6232c1aa7bb541a5bd667ae45f0f0c1c is 50, key is test_row_2/C:col10/1731187053456/Put/seqid=0 2024-11-09T21:17:36,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742264_1440 (size=7415) 2024-11-09T21:17:36,755 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=316 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/6232c1aa7bb541a5bd667ae45f0f0c1c 2024-11-09T21:17:36,760 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/A/347eaf6212274208a8d0521f5b480971 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/347eaf6212274208a8d0521f5b480971 2024-11-09T21:17:36,764 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/347eaf6212274208a8d0521f5b480971, entries=50, sequenceid=316, filesize=13.5 K 2024-11-09T21:17:36,765 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/B/75a691680bf6421aaad9e985dad4b44b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/75a691680bf6421aaad9e985dad4b44b 2024-11-09T21:17:36,770 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/75a691680bf6421aaad9e985dad4b44b, entries=50, sequenceid=316, filesize=7.2 K 2024-11-09T21:17:36,770 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/.tmp/C/6232c1aa7bb541a5bd667ae45f0f0c1c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6232c1aa7bb541a5bd667ae45f0f0c1c 2024-11-09T21:17:36,773 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6232c1aa7bb541a5bd667ae45f0f0c1c, entries=50, sequenceid=316, filesize=7.2 K 2024-11-09T21:17:36,774 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 010ea70b10f616a09e8971f27ee90066 in 1664ms, sequenceid=316, compaction requested=true 2024-11-09T21:17:36,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a42c455421334d16b4e4829933eaf695, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/2e1c68f651084caf8880a6e1b2c0a058, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/cb3b72d6c7164ee9b4ccae321adc07c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4f37d2b9f6ee4d82a69f8c5960350f76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc] to archive 2024-11-09T21:17:36,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:17:36,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/acaf6075485e40d7964ad856b1c2ad31 2024-11-09T21:17:36,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/8fe3f2e4bb414ac087d599d0a1ce02a3 2024-11-09T21:17:36,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f7db0c63997f4e1280fc30b8c89739c9 2024-11-09T21:17:36,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a42c455421334d16b4e4829933eaf695 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a42c455421334d16b4e4829933eaf695 2024-11-09T21:17:36,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c298654d0261441c8624613e51fdf0e7 2024-11-09T21:17:36,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/ddaa68020ca8476abcebdfb41dc51acc 2024-11-09T21:17:36,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/2e1c68f651084caf8880a6e1b2c0a058 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/2e1c68f651084caf8880a6e1b2c0a058 2024-11-09T21:17:36,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/712089d9c19b4e35b9c00a3b44639b0a 2024-11-09T21:17:36,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/52d6761d841647fcb80bb7220bdbdac8 2024-11-09T21:17:36,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/6d75a10f08434e9fbd7aff3183aef649 2024-11-09T21:17:36,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/cb3b72d6c7164ee9b4ccae321adc07c0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/cb3b72d6c7164ee9b4ccae321adc07c0 2024-11-09T21:17:36,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/a599f2dc9c1f4ea1a0a5f0dba10ab825 2024-11-09T21:17:36,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/c0d23f4f592443ae87e39a4e5332cef2 2024-11-09T21:17:36,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/050c55f5d8744f16a5df7b2ea77a43eb 2024-11-09T21:17:36,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/f51311f035e24d4dbd7b7983cca6ae20 2024-11-09T21:17:36,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/69e0af43d46e4cc1beca22ef3bd9b04e 2024-11-09T21:17:36,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4a1bfcd97ec2423d847a18ba7bb33068 2024-11-09T21:17:36,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4f37d2b9f6ee4d82a69f8c5960350f76 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4f37d2b9f6ee4d82a69f8c5960350f76 2024-11-09T21:17:36,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/4ae672a8e7ee44738524e2db3dbaff49 2024-11-09T21:17:36,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/7f011b976f3b4c3199961539aee840cc 2024-11-09T21:17:36,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a87d65ef3c4f44aab3b01127abd403c1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a7e3feb3e406491ab27303312bd27dcc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/df52354832fd4baebbc573d45b4a301a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9768fb466ee94665bcfc9d79ed71a02e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/d0915dbb1b8a4adea5e2af0de871d854, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e4046a44d55143998e1e1fc6128b2336, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/6d4414a047e44efca3fdad99205c77e4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/17170bf20934457681f8e50f5eb94fd8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/aae4b1c7af60403392913b95212f3874, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/61f999914ac9405a927cbc15bf36e160, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9d3c62affd26460682ce311d622f68e2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/baebb205995d446f97af496c37ef5b85, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/fed5b99cafce494bb0c47228af382762, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/efc72d520d4b48a3929a7318859f0f0f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e3fffde625a14a57b360bf54b71a8754, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e27e2ab450fc499d8c3965dcd9369609, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a82fde50bf5c492d85280ef98e9b41eb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/0839dfa7ba0842fcaf66af4594d96f43, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/893de30f03d0464db515736d564de1e9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/985ca91f738844d8b3aa8d7461d953e3] to archive 2024-11-09T21:17:36,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:17:36,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a87d65ef3c4f44aab3b01127abd403c1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a87d65ef3c4f44aab3b01127abd403c1 2024-11-09T21:17:36,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a7e3feb3e406491ab27303312bd27dcc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a7e3feb3e406491ab27303312bd27dcc 2024-11-09T21:17:36,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/df52354832fd4baebbc573d45b4a301a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/df52354832fd4baebbc573d45b4a301a 2024-11-09T21:17:36,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9768fb466ee94665bcfc9d79ed71a02e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9768fb466ee94665bcfc9d79ed71a02e 2024-11-09T21:17:36,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/d0915dbb1b8a4adea5e2af0de871d854 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/d0915dbb1b8a4adea5e2af0de871d854 2024-11-09T21:17:36,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e4046a44d55143998e1e1fc6128b2336 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e4046a44d55143998e1e1fc6128b2336 2024-11-09T21:17:36,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/6d4414a047e44efca3fdad99205c77e4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/6d4414a047e44efca3fdad99205c77e4 2024-11-09T21:17:36,808 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/17170bf20934457681f8e50f5eb94fd8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/17170bf20934457681f8e50f5eb94fd8 2024-11-09T21:17:36,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/aae4b1c7af60403392913b95212f3874 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/aae4b1c7af60403392913b95212f3874 2024-11-09T21:17:36,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/61f999914ac9405a927cbc15bf36e160 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/61f999914ac9405a927cbc15bf36e160 2024-11-09T21:17:36,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9d3c62affd26460682ce311d622f68e2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/9d3c62affd26460682ce311d622f68e2 2024-11-09T21:17:36,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/baebb205995d446f97af496c37ef5b85 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/baebb205995d446f97af496c37ef5b85 2024-11-09T21:17:36,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/fed5b99cafce494bb0c47228af382762 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/fed5b99cafce494bb0c47228af382762 2024-11-09T21:17:36,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/efc72d520d4b48a3929a7318859f0f0f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/efc72d520d4b48a3929a7318859f0f0f 2024-11-09T21:17:36,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e3fffde625a14a57b360bf54b71a8754 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e3fffde625a14a57b360bf54b71a8754 2024-11-09T21:17:36,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e27e2ab450fc499d8c3965dcd9369609 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/e27e2ab450fc499d8c3965dcd9369609 2024-11-09T21:17:36,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a82fde50bf5c492d85280ef98e9b41eb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/a82fde50bf5c492d85280ef98e9b41eb 2024-11-09T21:17:36,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/0839dfa7ba0842fcaf66af4594d96f43 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/0839dfa7ba0842fcaf66af4594d96f43 2024-11-09T21:17:36,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/893de30f03d0464db515736d564de1e9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/893de30f03d0464db515736d564de1e9 2024-11-09T21:17:36,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/985ca91f738844d8b3aa8d7461d953e3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/985ca91f738844d8b3aa8d7461d953e3 2024-11-09T21:17:36,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/298f4da6891e4ce58be3b32b6fb70d35, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6efb403b53534b9cb24bd0041aeab76b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/df48806a4ff645038f539324e9cf3e44, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/e1337d484f8c487f9b5a7f87a1999039, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/73c2c21a33eb4f8b8151757c53131867, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/04a14186c5f64e8c952bc673b7cb69c8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/4b0400f012a24864af923f567a02b931, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/fc5dfcbe9f0e45879ca6ae0f385b92ae, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7adaa074e22c42f59ab026a3ade5075e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/c0038b1e0bbe48daa5c99c3befe3f3e9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/681ce7c74a4c4047b9d69ceba3d52c62, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7b4c08ccf5994dc58ea5557d62622627, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/721aa5bca36f4b5dab088d769b9ccc34, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ee4d8dd6d1cb40d4bac2d64162487f09, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/60ab3481cb9d4579bb9b4e0d4f0eac1d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ed62bef3a6984e72bafc1e52535ac917, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/812717c7221c41e89ca567b34de8bea1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/248701ed3fa3417a92fd9acc3cc27702, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/aa1c92a811f0434481321f86757e3040, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/03d47e8f633b4820866bbdd37d7cf32b] to archive 2024-11-09T21:17:36,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:17:36,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/298f4da6891e4ce58be3b32b6fb70d35 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/298f4da6891e4ce58be3b32b6fb70d35 2024-11-09T21:17:36,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6efb403b53534b9cb24bd0041aeab76b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6efb403b53534b9cb24bd0041aeab76b 2024-11-09T21:17:36,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/df48806a4ff645038f539324e9cf3e44 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/df48806a4ff645038f539324e9cf3e44 2024-11-09T21:17:36,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/e1337d484f8c487f9b5a7f87a1999039 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/e1337d484f8c487f9b5a7f87a1999039 2024-11-09T21:17:36,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/73c2c21a33eb4f8b8151757c53131867 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/73c2c21a33eb4f8b8151757c53131867 2024-11-09T21:17:36,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/04a14186c5f64e8c952bc673b7cb69c8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/04a14186c5f64e8c952bc673b7cb69c8 2024-11-09T21:17:36,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/4b0400f012a24864af923f567a02b931 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/4b0400f012a24864af923f567a02b931 2024-11-09T21:17:36,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/fc5dfcbe9f0e45879ca6ae0f385b92ae to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/fc5dfcbe9f0e45879ca6ae0f385b92ae 2024-11-09T21:17:36,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7adaa074e22c42f59ab026a3ade5075e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7adaa074e22c42f59ab026a3ade5075e 2024-11-09T21:17:36,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/c0038b1e0bbe48daa5c99c3befe3f3e9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/c0038b1e0bbe48daa5c99c3befe3f3e9 2024-11-09T21:17:36,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/681ce7c74a4c4047b9d69ceba3d52c62 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/681ce7c74a4c4047b9d69ceba3d52c62 2024-11-09T21:17:36,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7b4c08ccf5994dc58ea5557d62622627 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/7b4c08ccf5994dc58ea5557d62622627 2024-11-09T21:17:36,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/721aa5bca36f4b5dab088d769b9ccc34 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/721aa5bca36f4b5dab088d769b9ccc34 2024-11-09T21:17:36,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ee4d8dd6d1cb40d4bac2d64162487f09 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ee4d8dd6d1cb40d4bac2d64162487f09 2024-11-09T21:17:36,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/60ab3481cb9d4579bb9b4e0d4f0eac1d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/60ab3481cb9d4579bb9b4e0d4f0eac1d 2024-11-09T21:17:36,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ed62bef3a6984e72bafc1e52535ac917 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/ed62bef3a6984e72bafc1e52535ac917 2024-11-09T21:17:36,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/812717c7221c41e89ca567b34de8bea1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/812717c7221c41e89ca567b34de8bea1 2024-11-09T21:17:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/248701ed3fa3417a92fd9acc3cc27702 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/248701ed3fa3417a92fd9acc3cc27702 2024-11-09T21:17:36,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/aa1c92a811f0434481321f86757e3040 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/aa1c92a811f0434481321f86757e3040 2024-11-09T21:17:36,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/03d47e8f633b4820866bbdd37d7cf32b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/03d47e8f633b4820866bbdd37d7cf32b 2024-11-09T21:17:36,845 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/recovered.edits/319.seqid, newMaxSeqId=319, maxSeqId=4 2024-11-09T21:17:36,845 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. 2024-11-09T21:17:36,845 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] regionserver.HRegion(1676): Region close journal for 010ea70b10f616a09e8971f27ee90066: Waiting for close lock at 1731187055110Running coprocessor pre-close hooks at 1731187055110Disabling compacts and flushes for region at 1731187055110Disabling writes for close at 1731187055110Obtaining lock to block concurrent updates at 1731187055110Preparing flush snapshotting stores in 010ea70b10f616a09e8971f27ee90066 at 1731187055110Finished memstore snapshotting TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731187055111 (+1 ms)Flushing stores of TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066. at 1731187055112 (+1 ms)Flushing 010ea70b10f616a09e8971f27ee90066/A: creating writer at 1731187055112Flushing mob file 010ea70b10f616a09e8971f27ee90066/A: appending metadata at 1731187055120 (+8 ms)Flushing mob file 010ea70b10f616a09e8971f27ee90066/A: closing flushed file at 1731187055120Flushing 010ea70b10f616a09e8971f27ee90066/A: appending metadata at 1731187055532 (+412 ms)Flushing 010ea70b10f616a09e8971f27ee90066/A: closing flushed file at 1731187055532Flushing 010ea70b10f616a09e8971f27ee90066/B: creating writer at 1731187055941 (+409 ms)Flushing 010ea70b10f616a09e8971f27ee90066/B: appending metadata at 1731187055943 (+2 ms)Flushing 010ea70b10f616a09e8971f27ee90066/B: closing flushed file at 1731187055943Flushing 010ea70b10f616a09e8971f27ee90066/C: creating writer at 1731187056349 (+406 ms)Flushing 010ea70b10f616a09e8971f27ee90066/C: appending metadata at 1731187056351 (+2 ms)Flushing 010ea70b10f616a09e8971f27ee90066/C: closing flushed file at 1731187056351Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5b9bba11: reopening flushed file at 1731187056759 (+408 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4361ffb5: reopening flushed file at 1731187056764 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@599f4814: reopening flushed file at 1731187056770 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 010ea70b10f616a09e8971f27ee90066 in 1664ms, sequenceid=316, compaction requested=true at 1731187056774 (+4 ms)Writing region close event to WAL at 1731187056842 (+68 ms)Running coprocessor post-close hooks at 1731187056845 (+3 ms)Closed at 1731187056845 2024-11-09T21:17:36,847 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=122}] handler.UnassignRegionHandler(157): Closed 010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:36,848 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=121 updating hbase:meta row=010ea70b10f616a09e8971f27ee90066, regionState=CLOSED 2024-11-09T21:17:36,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=122, ppid=121, state=RUNNABLE, hasLock=false; CloseRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:17:36,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-09T21:17:36,851 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; CloseRegionProcedure 010ea70b10f616a09e8971f27ee90066, server=9514d6d5e5be,38413,1731186934857 in 1.8920 sec 2024-11-09T21:17:36,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=121, resume processing ppid=120 2024-11-09T21:17:36,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, ppid=120, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=010ea70b10f616a09e8971f27ee90066, UNASSIGN in 1.8970 sec 2024-11-09T21:17:36,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-09T21:17:36,854 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9000 sec 2024-11-09T21:17:36,855 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187056855"}]},"ts":"1731187056855"} 2024-11-09T21:17:36,856 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T21:17:36,856 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T21:17:36,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9080 sec 2024-11-09T21:17:37,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-09T21:17:37,107 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:37,107 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T21:17:37,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,110 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=123, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T21:17:37,110 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=123, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,113 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,116 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/recovered.edits] 2024-11-09T21:17:37,120 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/29474628994d41d2883b25a4e38eb0cf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/29474628994d41d2883b25a4e38eb0cf 2024-11-09T21:17:37,122 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/347eaf6212274208a8d0521f5b480971 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/347eaf6212274208a8d0521f5b480971 2024-11-09T21:17:37,123 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/34ef54a398ee4914bbc4092f95a1d541 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/A/34ef54a398ee4914bbc4092f95a1d541 2024-11-09T21:17:37,126 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/64aafd3d32664fc8bec187e0e33a43ba to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/64aafd3d32664fc8bec187e0e33a43ba 2024-11-09T21:17:37,127 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/75a691680bf6421aaad9e985dad4b44b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/75a691680bf6421aaad9e985dad4b44b 2024-11-09T21:17:37,127 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/bdaf78c35c2444319889b739831b665d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/B/bdaf78c35c2444319889b739831b665d 2024-11-09T21:17:37,129 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6232c1aa7bb541a5bd667ae45f0f0c1c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/6232c1aa7bb541a5bd667ae45f0f0c1c 2024-11-09T21:17:37,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/95435a01492e4412875fe1c89f2f920c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/95435a01492e4412875fe1c89f2f920c 2024-11-09T21:17:37,130 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/b407d45c476e4756b044563ff65ea18b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/C/b407d45c476e4756b044563ff65ea18b 2024-11-09T21:17:37,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/recovered.edits/319.seqid to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066/recovered.edits/319.seqid 2024-11-09T21:17:37,133 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,133 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T21:17:37,133 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T21:17:37,134 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T21:17:37,137 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d4b2721c99541788527a8920650d728_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090d4b2721c99541788527a8920650d728_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,138 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090ea55c94275a4d46a34c4618c0e3f631_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411090ea55c94275a4d46a34c4618c0e3f631_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,139 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110910141a551e08419da08f878192b5e89d_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110910141a551e08419da08f878192b5e89d_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,140 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093197caf837ac4abdb4cf839dfa02b118_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093197caf837ac4abdb4cf839dfa02b118_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,141 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110947adf340ed2c4b588932c204bf28f4d6_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110947adf340ed2c4b588932c204bf28f4d6_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,142 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110948db5144ec874f0d881d5a98d5c88946_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110948db5144ec874f0d881d5a98d5c88946_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,143 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110953af2db654344a368fa5ef2b20865145_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110953af2db654344a368fa5ef2b20865145_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,144 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110955fe2d64e36a4046ade1cada9c5021f2_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024110955fe2d64e36a4046ade1cada9c5021f2_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,144 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099bee153832c247b6bfc9300a16b15c54_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099bee153832c247b6bfc9300a16b15c54_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,145 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a10bffff5dab48699f251c1afc720c3d_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a10bffff5dab48699f251c1afc720c3d_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,146 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a5c130acac5a4dc9bca4e9a0243d56d1_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a5c130acac5a4dc9bca4e9a0243d56d1_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,146 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a61743720b984aabb154cf27a790ce08_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a61743720b984aabb154cf27a790ce08_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,147 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a746df054a284e71977d8e6de0211322_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a746df054a284e71977d8e6de0211322_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,148 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a923b43f1c26415c958da2965ce7ac5a_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a923b43f1c26415c958da2965ce7ac5a_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,149 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b46f36219cb2487bb523aeea1dd54604_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b46f36219cb2487bb523aeea1dd54604_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,149 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c8a40a6089e64b7b97703ea9307a20cc_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c8a40a6089e64b7b97703ea9307a20cc_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,150 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109de9742023765497faf092682463bd23b_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109de9742023765497faf092682463bd23b_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,151 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ff0e0ea151764621a71399ae7edd56bd_010ea70b10f616a09e8971f27ee90066 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109ff0e0ea151764621a71399ae7edd56bd_010ea70b10f616a09e8971f27ee90066 2024-11-09T21:17:37,151 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T21:17:37,153 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=123, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,155 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T21:17:37,157 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T21:17:37,158 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=123, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,158 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T21:17:37,158 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731187057158"}]},"ts":"9223372036854775807"} 2024-11-09T21:17:37,159 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T21:17:37,160 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 010ea70b10f616a09e8971f27ee90066, NAME => 'TestAcidGuarantees,,1731187029287.010ea70b10f616a09e8971f27ee90066.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T21:17:37,160 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T21:17:37,160 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731187057160"}]},"ts":"9223372036854775807"} 2024-11-09T21:17:37,161 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T21:17:37,162 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=123, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 55 msec 2024-11-09T21:17:37,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-09T21:17:37,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T21:17:37,218 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:37,233 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=251 (was 246) - Thread LEAK? -, OpenFileDescriptor=459 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 368) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3589 (was 2983) - AvailableMemoryMB LEAK? - 2024-11-09T21:17:37,242 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=3589 2024-11-09T21:17:37,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:17:37,244 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:17:37,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T21:17:37,246 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:17:37,246 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:37,246 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 124 2024-11-09T21:17:37,246 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:17:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-09T21:17:37,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742265_1441 (size=1061) 2024-11-09T21:17:37,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-09T21:17:37,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-09T21:17:37,652 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:17:37,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742266_1442 (size=53) 2024-11-09T21:17:37,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-09T21:17:38,058 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:17:38,058 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2b19020342a14d63e946c9297523dd88, disabling compactions & flushes 2024-11-09T21:17:38,058 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,058 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,058 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. after waiting 0 ms 2024-11-09T21:17:38,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,059 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,059 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2b19020342a14d63e946c9297523dd88: Waiting for close lock at 1731187058058Disabling compacts and flushes for region at 1731187058058Disabling writes for close at 1731187058058Writing region close event to WAL at 1731187058059 (+1 ms)Closed at 1731187058059 2024-11-09T21:17:38,060 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:17:38,060 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731187058060"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187058060"}]},"ts":"1731187058060"} 2024-11-09T21:17:38,061 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:17:38,062 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:17:38,062 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187058062"}]},"ts":"1731187058062"} 2024-11-09T21:17:38,063 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T21:17:38,063 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, ASSIGN}] 2024-11-09T21:17:38,064 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, ASSIGN 2024-11-09T21:17:38,065 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, ASSIGN; state=OFFLINE, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=false 2024-11-09T21:17:38,215 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2b19020342a14d63e946c9297523dd88, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:38,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, ASSIGN because future has completed 2024-11-09T21:17:38,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:17:38,371 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,371 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7752): Opening region: {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:17:38,372 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,372 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:17:38,372 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7794): checking encryption for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,372 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(7797): checking classloading for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,374 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,375 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:38,376 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b19020342a14d63e946c9297523dd88 columnFamilyName A 2024-11-09T21:17:38,376 DEBUG [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:38,376 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.HStore(327): Store=2b19020342a14d63e946c9297523dd88/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:38,376 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,378 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:38,378 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b19020342a14d63e946c9297523dd88 columnFamilyName B 2024-11-09T21:17:38,378 DEBUG [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:38,379 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.HStore(327): Store=2b19020342a14d63e946c9297523dd88/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:38,379 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,380 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:17:38,380 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2b19020342a14d63e946c9297523dd88 columnFamilyName C 2024-11-09T21:17:38,381 DEBUG [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:17:38,381 INFO [StoreOpener-2b19020342a14d63e946c9297523dd88-1 {}] regionserver.HStore(327): Store=2b19020342a14d63e946c9297523dd88/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:17:38,381 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,382 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1038): replaying wal for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,383 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,383 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,384 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1048): stopping wal replay for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,385 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1060): Cleaning up temporary data for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,385 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:17:38,386 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1093): writing seq id for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-09T21:17:38,388 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:17:38,389 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1114): Opened 2b19020342a14d63e946c9297523dd88; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63347002, jitterRate=-0.05605611205101013}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:17:38,389 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:38,390 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegion(1006): Region open journal for 2b19020342a14d63e946c9297523dd88: Running coprocessor pre-open hook at 1731187058372Writing region info on filesystem at 1731187058372Initializing all the Stores at 1731187058373 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187058373Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187058374 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187058374Cleaning up temporary data from old regions at 1731187058385 (+11 ms)Running coprocessor post-open hooks at 1731187058389 (+4 ms)Region opened successfully at 1731187058389 2024-11-09T21:17:38,390 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., pid=126, masterSystemTime=1731187058368 2024-11-09T21:17:38,392 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,392 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=126}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:38,393 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=2b19020342a14d63e946c9297523dd88, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:38,394 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:17:38,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-09T21:17:38,396 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; OpenRegionProcedure 2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 in 178 msec 2024-11-09T21:17:38,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-09T21:17:38,398 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, ASSIGN in 333 msec 2024-11-09T21:17:38,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:17:38,398 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187058398"}]},"ts":"1731187058398"} 2024-11-09T21:17:38,399 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T21:17:38,400 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=124, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:17:38,401 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-09T21:17:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-11-09T21:17:39,397 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:39,398 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3adc9e66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,398 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,398 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,399 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ec8522d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,400 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,401 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42994, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,402 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef85509, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,402 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,403 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60454, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,406 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:17:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:17:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3c5cbde2 2024-11-09T21:17:39,407 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:17:39,408 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42996, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:17:39,409 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@720699be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,409 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,409 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,410 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,410 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@726ab668, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,411 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43026, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,412 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe8cf2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,413 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@119ad987, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,413 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5884517f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,415 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43048, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,415 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fde8ae6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,416 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26c4ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,416 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,416 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,417 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e75073, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,417 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,418 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43068, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4524b009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd1f545, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,420 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,421 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39d0802c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,422 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43086, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@506d79bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,423 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c19380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,423 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,423 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,425 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12cf916b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,425 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,426 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43108, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e6ca402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,427 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1627c26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,427 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,427 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,428 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48d50cb9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,429 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,430 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@780af1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5993889b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,431 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e41a32c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,432 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,433 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,433 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68be0394, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,434 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@117399b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,434 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,434 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,435 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cfff2c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,436 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,437 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43166, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,438 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ac7a8e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,438 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b33a176, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,438 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,439 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,439 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50486ea1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,441 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43192, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,441 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73d638fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:17:39,442 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:17:39,443 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:17:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:17:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:17:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@178ef34, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:17:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:17:39,443 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:39,444 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:17:39,445 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ba9da63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:17:39,445 DEBUG [Thread-1993 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,445 DEBUG [Thread-1998 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,445 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:39,446 DEBUG [Thread-1987 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,446 DEBUG [Thread-1991 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-09T21:17:39,448 DEBUG [Thread-1989 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,448 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,448 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60460, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,449 DEBUG [Thread-1989 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,449 DEBUG [Thread-1989 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,449 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60480, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,449 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:39,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:39,450 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,450 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,451 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:39,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,453 DEBUG [Thread-1995 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,454 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,457 DEBUG [Thread-2006 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,458 DEBUG [Thread-2002 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,459 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60522, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,460 DEBUG [Thread-2000 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,461 DEBUG [Thread-2004 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:17:39,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:39,461 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60536, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,462 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:17:39,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:17:39,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,463 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:17:39,464 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60538, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60542, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:17:39,466 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2] 2024-11-09T21:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:39,467 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:39,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/3d57fbbfd47a4d41af43e3ee4c49a2ae is 50, key is test_row_0/A:col10/1731187059458/Put/seqid=0 2024-11-09T21:17:39,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742267_1443 (size=16681) 2024-11-09T21:17:39,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/3d57fbbfd47a4d41af43e3ee4c49a2ae 2024-11-09T21:17:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187119497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187119498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187119500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187119501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187119508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/db61fe608902470ebac9fff5f0f6669f is 50, key is test_row_0/B:col10/1731187059458/Put/seqid=0 2024-11-09T21:17:39,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742268_1444 (size=12001) 2024-11-09T21:17:39,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:39,605 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:39,606 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187119616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187119616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187119616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187119616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187119627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,758 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:39,758 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:39,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187119826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187119826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,834 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187119826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,834 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,835 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187119836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:39,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187119846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:39,851 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:39,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:39,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:39,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:39,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:39,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:39,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/db61fe608902470ebac9fff5f0f6669f 2024-11-09T21:17:39,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8f0e07846f13469e81874e96b6d776d1 is 50, key is test_row_0/C:col10/1731187059458/Put/seqid=0 2024-11-09T21:17:39,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742269_1445 (size=12001) 2024-11-09T21:17:40,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:40,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187120136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187120146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,152 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187120146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187120146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,153 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187120156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,162 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8f0e07846f13469e81874e96b6d776d1 2024-11-09T21:17:40,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/3d57fbbfd47a4d41af43e3ee4c49a2ae as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3d57fbbfd47a4d41af43e3ee4c49a2ae 2024-11-09T21:17:40,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,369 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:40,370 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3d57fbbfd47a4d41af43e3ee4c49a2ae, entries=250, sequenceid=15, filesize=16.3 K 2024-11-09T21:17:40,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/db61fe608902470ebac9fff5f0f6669f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/db61fe608902470ebac9fff5f0f6669f 2024-11-09T21:17:40,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/db61fe608902470ebac9fff5f0f6669f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T21:17:40,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8f0e07846f13469e81874e96b6d776d1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f0e07846f13469e81874e96b6d776d1 2024-11-09T21:17:40,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f0e07846f13469e81874e96b6d776d1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-09T21:17:40,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2b19020342a14d63e946c9297523dd88 in 915ms, sequenceid=15, compaction requested=false 2024-11-09T21:17:40,377 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-09T21:17:40,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:40,521 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:40,521 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:40,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:40,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/6065ab717b2b47f9b4024666e8fda281 is 50, key is test_row_0/A:col10/1731187059480/Put/seqid=0 2024-11-09T21:17:40,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742270_1446 (size=12001) 2024-11-09T21:17:40,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:40,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:40,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:40,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187120669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187120671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,678 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187120672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,678 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187120674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187120675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187120786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187120786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187120786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187120786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187120786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:40,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:40,929 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/6065ab717b2b47f9b4024666e8fda281 2024-11-09T21:17:40,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/c9ebe5b570e24b07a83a795e73a6efab is 50, key is test_row_0/B:col10/1731187059480/Put/seqid=0 2024-11-09T21:17:40,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742271_1447 (size=12001) 2024-11-09T21:17:41,012 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187121007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187121007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187121007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187121008, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,014 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187121012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187121316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187121326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,332 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187121326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,332 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187121326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187121326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,333 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/c9ebe5b570e24b07a83a795e73a6efab 2024-11-09T21:17:41,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/9dad551c1dce4f8eb6990829c915ffe1 is 50, key is test_row_0/C:col10/1731187059480/Put/seqid=0 2024-11-09T21:17:41,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742272_1448 (size=12001) 2024-11-09T21:17:41,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:41,750 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/9dad551c1dce4f8eb6990829c915ffe1 2024-11-09T21:17:41,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/6065ab717b2b47f9b4024666e8fda281 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/6065ab717b2b47f9b4024666e8fda281 2024-11-09T21:17:41,759 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/6065ab717b2b47f9b4024666e8fda281, entries=150, sequenceid=33, filesize=11.7 K 2024-11-09T21:17:41,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/c9ebe5b570e24b07a83a795e73a6efab as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c9ebe5b570e24b07a83a795e73a6efab 2024-11-09T21:17:41,767 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c9ebe5b570e24b07a83a795e73a6efab, entries=150, sequenceid=33, filesize=11.7 K 2024-11-09T21:17:41,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/9dad551c1dce4f8eb6990829c915ffe1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9dad551c1dce4f8eb6990829c915ffe1 2024-11-09T21:17:41,770 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9dad551c1dce4f8eb6990829c915ffe1, entries=150, sequenceid=33, filesize=11.7 K 2024-11-09T21:17:41,771 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2b19020342a14d63e946c9297523dd88 in 1250ms, sequenceid=33, compaction requested=false 2024-11-09T21:17:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:41,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-09T21:17:41,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-09T21:17:41,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-09T21:17:41,774 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3200 sec 2024-11-09T21:17:41,776 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.3300 sec 2024-11-09T21:17:41,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:41,829 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:41,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:41,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:41,830 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:41,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/ce5fcf9aba1f4b8f859832e1b943d5df is 50, key is test_row_0/A:col10/1731187060670/Put/seqid=0 2024-11-09T21:17:41,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742273_1449 (size=14341) 2024-11-09T21:17:41,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187121853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,867 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187121854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,868 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187121861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,868 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,869 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187121862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187121865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187121976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187121976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,980 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187121976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187121986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:41,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:41,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187121986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:41,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187122186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187122186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187122186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187122196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187122196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/ce5fcf9aba1f4b8f859832e1b943d5df 2024-11-09T21:17:42,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ed471fb79226427bbcd24c6123d46358 is 50, key is test_row_0/B:col10/1731187060670/Put/seqid=0 2024-11-09T21:17:42,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742274_1450 (size=12001) 2024-11-09T21:17:42,369 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:17:42,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187122496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187122496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,506 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187122496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,506 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187122506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:42,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187122508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:42,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:42,650 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ed471fb79226427bbcd24c6123d46358 2024-11-09T21:17:42,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/98474b11e4374f92acdeae72f6cf93a8 is 50, key is test_row_0/C:col10/1731187060670/Put/seqid=0 2024-11-09T21:17:42,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742275_1451 (size=12001) 2024-11-09T21:17:43,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:43,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187123017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:43,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187123018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:43,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187123018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,031 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187123026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187123026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:43,033 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:43,034 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:43,058 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/98474b11e4374f92acdeae72f6cf93a8 2024-11-09T21:17:43,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/ce5fcf9aba1f4b8f859832e1b943d5df as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/ce5fcf9aba1f4b8f859832e1b943d5df 2024-11-09T21:17:43,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/ce5fcf9aba1f4b8f859832e1b943d5df, entries=200, sequenceid=48, filesize=14.0 K 2024-11-09T21:17:43,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ed471fb79226427bbcd24c6123d46358 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ed471fb79226427bbcd24c6123d46358 2024-11-09T21:17:43,074 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ed471fb79226427bbcd24c6123d46358, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T21:17:43,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/98474b11e4374f92acdeae72f6cf93a8 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/98474b11e4374f92acdeae72f6cf93a8 2024-11-09T21:17:43,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/98474b11e4374f92acdeae72f6cf93a8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-09T21:17:43,083 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2b19020342a14d63e946c9297523dd88 in 1254ms, sequenceid=48, compaction requested=true 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:43,083 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:43,083 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:43,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:43,084 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:43,084 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43023 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:43,084 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:43,084 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:43,084 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:43,084 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:43,084 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/db61fe608902470ebac9fff5f0f6669f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c9ebe5b570e24b07a83a795e73a6efab, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ed471fb79226427bbcd24c6123d46358] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.2 K 2024-11-09T21:17:43,085 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3d57fbbfd47a4d41af43e3ee4c49a2ae, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/6065ab717b2b47f9b4024666e8fda281, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/ce5fcf9aba1f4b8f859832e1b943d5df] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=42.0 K 2024-11-09T21:17:43,085 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting db61fe608902470ebac9fff5f0f6669f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731187059458 2024-11-09T21:17:43,085 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d57fbbfd47a4d41af43e3ee4c49a2ae, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731187059457 2024-11-09T21:17:43,089 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ebe5b570e24b07a83a795e73a6efab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731187059478 2024-11-09T21:17:43,089 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6065ab717b2b47f9b4024666e8fda281, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731187059478 2024-11-09T21:17:43,090 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ed471fb79226427bbcd24c6123d46358, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731187060660 2024-11-09T21:17:43,090 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5fcf9aba1f4b8f859832e1b943d5df, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731187060660 2024-11-09T21:17:43,103 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#379 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:43,103 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:43,103 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/9e5bb3827c064e0fa85dbc81b12cfa55 is 50, key is test_row_0/A:col10/1731187060670/Put/seqid=0 2024-11-09T21:17:43,103 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/072e320d602e4843b24efc28450e0053 is 50, key is test_row_0/B:col10/1731187060670/Put/seqid=0 2024-11-09T21:17:43,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742276_1452 (size=12104) 2024-11-09T21:17:43,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742277_1453 (size=12104) 2024-11-09T21:17:43,123 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/9e5bb3827c064e0fa85dbc81b12cfa55 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9e5bb3827c064e0fa85dbc81b12cfa55 2024-11-09T21:17:43,123 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/072e320d602e4843b24efc28450e0053 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/072e320d602e4843b24efc28450e0053 2024-11-09T21:17:43,132 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into 072e320d602e4843b24efc28450e0053(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:43,132 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into 9e5bb3827c064e0fa85dbc81b12cfa55(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:43,132 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187063083; duration=0sec 2024-11-09T21:17:43,132 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187063083; duration=0sec 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:43,132 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:43,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:43,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:43,133 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:43,133 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f0e07846f13469e81874e96b6d776d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9dad551c1dce4f8eb6990829c915ffe1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/98474b11e4374f92acdeae72f6cf93a8] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.2 K 2024-11-09T21:17:43,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f0e07846f13469e81874e96b6d776d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731187059458 2024-11-09T21:17:43,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dad551c1dce4f8eb6990829c915ffe1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=33, earliestPutTs=1731187059478 2024-11-09T21:17:43,134 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98474b11e4374f92acdeae72f6cf93a8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731187060660 2024-11-09T21:17:43,144 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#380 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:43,144 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/b118933bb9f7452194451e7d2fa20e03 is 50, key is test_row_0/C:col10/1731187060670/Put/seqid=0 2024-11-09T21:17:43,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742278_1454 (size=12104) 2024-11-09T21:17:43,153 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/b118933bb9f7452194451e7d2fa20e03 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b118933bb9f7452194451e7d2fa20e03 2024-11-09T21:17:43,157 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into b118933bb9f7452194451e7d2fa20e03(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:43,157 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:43,157 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187063083; duration=0sec 2024-11-09T21:17:43,157 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:43,157 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:43,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-09T21:17:43,617 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:43,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-09T21:17:43,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T21:17:43,618 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:43,619 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:43,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:43,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T21:17:43,772 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-09T21:17:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:43,772 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:43,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:43,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:43,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/39e25a524d8046b2bf5eab0ca98b855c is 50, key is test_row_0/A:col10/1731187061852/Put/seqid=0 2024-11-09T21:17:43,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742279_1455 (size=12001) 2024-11-09T21:17:43,799 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/39e25a524d8046b2bf5eab0ca98b855c 2024-11-09T21:17:43,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/98806f82e69f4b7ca849e3ee2cda6050 is 50, key is test_row_0/B:col10/1731187061852/Put/seqid=0 2024-11-09T21:17:43,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742280_1456 (size=12001) 2024-11-09T21:17:43,822 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/98806f82e69f4b7ca849e3ee2cda6050 2024-11-09T21:17:43,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/76b41ded0c1649d18a2ed94c92f6b74c is 50, key is test_row_0/C:col10/1731187061852/Put/seqid=0 2024-11-09T21:17:43,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742281_1457 (size=12001) 2024-11-09T21:17:43,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T21:17:44,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:44,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187124053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187124054, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,063 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187124055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187124059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,065 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,069 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187124063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187124166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187124166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187124177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187124177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,185 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187124177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,186 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,241 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=68 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/76b41ded0c1649d18a2ed94c92f6b74c 2024-11-09T21:17:44,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/39e25a524d8046b2bf5eab0ca98b855c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/39e25a524d8046b2bf5eab0ca98b855c 2024-11-09T21:17:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T21:17:44,247 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/39e25a524d8046b2bf5eab0ca98b855c, entries=150, sequenceid=68, filesize=11.7 K 2024-11-09T21:17:44,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/98806f82e69f4b7ca849e3ee2cda6050 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/98806f82e69f4b7ca849e3ee2cda6050 2024-11-09T21:17:44,250 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/98806f82e69f4b7ca849e3ee2cda6050, entries=150, sequenceid=68, filesize=11.7 K 2024-11-09T21:17:44,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/76b41ded0c1649d18a2ed94c92f6b74c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/76b41ded0c1649d18a2ed94c92f6b74c 2024-11-09T21:17:44,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/76b41ded0c1649d18a2ed94c92f6b74c, entries=150, sequenceid=68, filesize=11.7 K 2024-11-09T21:17:44,256 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2b19020342a14d63e946c9297523dd88 in 484ms, sequenceid=68, compaction requested=false 2024-11-09T21:17:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:44,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-09T21:17:44,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-09T21:17:44,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-09T21:17:44,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 638 msec 2024-11-09T21:17:44,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 642 msec 2024-11-09T21:17:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:44,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T21:17:44,383 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:44,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:44,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:44,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:44,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:44,384 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:44,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/cb4dbec5ede840dab0cfe8d1497853ef is 50, key is test_row_0/A:col10/1731187064062/Put/seqid=0 2024-11-09T21:17:44,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742282_1458 (size=14341) 2024-11-09T21:17:44,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/cb4dbec5ede840dab0cfe8d1497853ef 2024-11-09T21:17:44,403 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/9bd9acead8344dcd892a7d94b41bd523 is 50, key is test_row_0/B:col10/1731187064062/Put/seqid=0 2024-11-09T21:17:44,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742283_1459 (size=12001) 2024-11-09T21:17:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187124408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187124409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187124417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,423 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187124418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187124419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187124526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,533 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187124526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187124527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,534 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187124536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,546 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187124536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,546 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187124737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187124737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,747 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,750 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187124737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,750 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-09T21:17:44,756 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:44,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:44,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-09T21:17:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T21:17:44,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:44,758 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:44,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:44,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187124756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187124756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:44,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:44,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/9bd9acead8344dcd892a7d94b41bd523 2024-11-09T21:17:44,813 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/100d6af7f3ff4b279996f4d10c4cb907 is 50, key is test_row_0/C:col10/1731187064062/Put/seqid=0 2024-11-09T21:17:44,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742284_1460 (size=12001) 2024-11-09T21:17:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T21:17:44,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T21:17:44,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:44,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:44,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:44,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:44,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:44,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T21:17:45,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:45,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,064 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187125056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,067 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,067 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187125056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187125056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T21:17:45,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187125076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187125076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,086 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T21:17:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,216 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:45,222 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/100d6af7f3ff4b279996f4d10c4cb907 2024-11-09T21:17:45,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/cb4dbec5ede840dab0cfe8d1497853ef as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cb4dbec5ede840dab0cfe8d1497853ef 2024-11-09T21:17:45,228 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cb4dbec5ede840dab0cfe8d1497853ef, entries=200, sequenceid=84, filesize=14.0 K 2024-11-09T21:17:45,229 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/9bd9acead8344dcd892a7d94b41bd523 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9bd9acead8344dcd892a7d94b41bd523 2024-11-09T21:17:45,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9bd9acead8344dcd892a7d94b41bd523, entries=150, sequenceid=84, filesize=11.7 K 2024-11-09T21:17:45,232 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/100d6af7f3ff4b279996f4d10c4cb907 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/100d6af7f3ff4b279996f4d10c4cb907 2024-11-09T21:17:45,234 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/100d6af7f3ff4b279996f4d10c4cb907, entries=150, sequenceid=84, filesize=11.7 K 2024-11-09T21:17:45,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2b19020342a14d63e946c9297523dd88 in 851ms, sequenceid=84, compaction requested=true 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:45,235 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:45,235 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:45,235 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:45,235 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:45,235 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:45,235 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:45,235 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:45,235 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,235 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,235 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9e5bb3827c064e0fa85dbc81b12cfa55, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/39e25a524d8046b2bf5eab0ca98b855c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cb4dbec5ede840dab0cfe8d1497853ef] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=37.5 K 2024-11-09T21:17:45,235 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/072e320d602e4843b24efc28450e0053, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/98806f82e69f4b7ca849e3ee2cda6050, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9bd9acead8344dcd892a7d94b41bd523] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.3 K 2024-11-09T21:17:45,236 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 072e320d602e4843b24efc28450e0053, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731187060660 2024-11-09T21:17:45,236 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e5bb3827c064e0fa85dbc81b12cfa55, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731187060660 2024-11-09T21:17:45,236 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 98806f82e69f4b7ca849e3ee2cda6050, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731187061851 2024-11-09T21:17:45,236 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39e25a524d8046b2bf5eab0ca98b855c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731187061851 2024-11-09T21:17:45,236 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd9acead8344dcd892a7d94b41bd523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731187064053 2024-11-09T21:17:45,236 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb4dbec5ede840dab0cfe8d1497853ef, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731187064053 2024-11-09T21:17:45,241 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:45,241 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/67aa496083ff4e9fac95c78914290bdf is 50, key is test_row_0/A:col10/1731187064062/Put/seqid=0 2024-11-09T21:17:45,244 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:45,244 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/c20bf16cbf684dcfb2d6c05d66ac4b76 is 50, key is test_row_0/B:col10/1731187064062/Put/seqid=0 2024-11-09T21:17:45,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742285_1461 (size=12207) 2024-11-09T21:17:45,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742286_1462 (size=12207) 2024-11-09T21:17:45,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-09T21:17:45,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,368 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T21:17:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:45,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:45,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a05a0bdbe73f4cbd81a301004feaa8f9 is 50, key is test_row_0/A:col10/1731187064418/Put/seqid=0 2024-11-09T21:17:45,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742287_1463 (size=12001) 2024-11-09T21:17:45,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a05a0bdbe73f4cbd81a301004feaa8f9 2024-11-09T21:17:45,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/a38726f3a99a4eea8ef27df31f88b5b3 is 50, key is test_row_0/B:col10/1731187064418/Put/seqid=0 2024-11-09T21:17:45,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742288_1464 (size=12001) 2024-11-09T21:17:45,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T21:17:45,387 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/a38726f3a99a4eea8ef27df31f88b5b3 2024-11-09T21:17:45,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/9fe2f32525884117bb18e9c3695b58e0 is 50, key is test_row_0/C:col10/1731187064418/Put/seqid=0 2024-11-09T21:17:45,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742289_1465 (size=12001) 2024-11-09T21:17:45,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:45,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187125601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,611 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187125602, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,614 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187125609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,614 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187125610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187125611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,622 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,652 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/67aa496083ff4e9fac95c78914290bdf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/67aa496083ff4e9fac95c78914290bdf 2024-11-09T21:17:45,656 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into 67aa496083ff4e9fac95c78914290bdf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:45,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:45,656 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187065235; duration=0sec 2024-11-09T21:17:45,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:45,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:45,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:45,657 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:45,657 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:45,657 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,657 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b118933bb9f7452194451e7d2fa20e03, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/76b41ded0c1649d18a2ed94c92f6b74c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/100d6af7f3ff4b279996f4d10c4cb907] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.3 K 2024-11-09T21:17:45,657 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b118933bb9f7452194451e7d2fa20e03, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731187060660 2024-11-09T21:17:45,657 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76b41ded0c1649d18a2ed94c92f6b74c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=68, earliestPutTs=1731187061851 2024-11-09T21:17:45,658 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100d6af7f3ff4b279996f4d10c4cb907, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731187064053 2024-11-09T21:17:45,659 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/c20bf16cbf684dcfb2d6c05d66ac4b76 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c20bf16cbf684dcfb2d6c05d66ac4b76 2024-11-09T21:17:45,662 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into c20bf16cbf684dcfb2d6c05d66ac4b76(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:45,662 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:45,662 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187065235; duration=0sec 2024-11-09T21:17:45,662 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:45,662 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:45,664 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:45,664 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8fda6af462a44411820007c99996f709 is 50, key is test_row_0/C:col10/1731187064062/Put/seqid=0 2024-11-09T21:17:45,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742290_1466 (size=12207) 2024-11-09T21:17:45,672 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8fda6af462a44411820007c99996f709 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8fda6af462a44411820007c99996f709 2024-11-09T21:17:45,676 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into 8fda6af462a44411820007c99996f709(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:45,676 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:45,676 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187065235; duration=0sec 2024-11-09T21:17:45,676 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:45,676 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187125716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187125716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187125716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,732 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187125727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,733 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187125727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,797 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/9fe2f32525884117bb18e9c3695b58e0 2024-11-09T21:17:45,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a05a0bdbe73f4cbd81a301004feaa8f9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a05a0bdbe73f4cbd81a301004feaa8f9 2024-11-09T21:17:45,803 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a05a0bdbe73f4cbd81a301004feaa8f9, entries=150, sequenceid=100, filesize=11.7 K 2024-11-09T21:17:45,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/a38726f3a99a4eea8ef27df31f88b5b3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a38726f3a99a4eea8ef27df31f88b5b3 2024-11-09T21:17:45,806 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a38726f3a99a4eea8ef27df31f88b5b3, entries=150, sequenceid=100, filesize=11.7 K 2024-11-09T21:17:45,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/9fe2f32525884117bb18e9c3695b58e0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9fe2f32525884117bb18e9c3695b58e0 2024-11-09T21:17:45,809 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9fe2f32525884117bb18e9c3695b58e0, entries=150, sequenceid=100, filesize=11.7 K 2024-11-09T21:17:45,810 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2b19020342a14d63e946c9297523dd88 in 442ms, sequenceid=100, compaction requested=false 2024-11-09T21:17:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:45,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-09T21:17:45,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-09T21:17:45,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-09T21:17:45,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0520 sec 2024-11-09T21:17:45,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.0560 sec 2024-11-09T21:17:45,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-09T21:17:45,896 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:45,897 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:45,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-09T21:17:45,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:45,898 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:45,899 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:45,899 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:45,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:45,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:45,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:45,938 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/3756a5c7fabe4905a9bae5f8909b8f4b is 50, key is test_row_0/A:col10/1731187065932/Put/seqid=0 2024-11-09T21:17:45,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742291_1467 (size=16681) 2024-11-09T21:17:45,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187125951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187125952, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,965 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187125964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187125964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,966 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:45,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:45,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187125965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:45,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:46,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,074 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187126066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187126076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187126077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187126077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,085 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187126086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187126286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187126286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187126286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187126297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,302 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187126306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,314 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,341 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/3756a5c7fabe4905a9bae5f8909b8f4b 2024-11-09T21:17:46,346 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/6b5056496dee43ebbb30a0cf73751f8b is 50, key is test_row_0/B:col10/1731187065932/Put/seqid=0 2024-11-09T21:17:46,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742292_1468 (size=12001) 2024-11-09T21:17:46,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:46,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187126596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187126606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187126606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187126606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:46,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187126616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:46,625 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:46,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/6b5056496dee43ebbb30a0cf73751f8b 2024-11-09T21:17:46,754 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/e38d25a6abdd4311bde8c14e60edb81d is 50, key is test_row_0/C:col10/1731187065932/Put/seqid=0 2024-11-09T21:17:46,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742293_1469 (size=12001) 2024-11-09T21:17:46,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:46,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:47,118 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:47,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:47,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:47,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:47,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187127117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187127117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,126 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:47,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187127117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:47,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187127117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:47,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:47,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187127136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:47,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=121 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/e38d25a6abdd4311bde8c14e60edb81d 2024-11-09T21:17:47,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/3756a5c7fabe4905a9bae5f8909b8f4b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3756a5c7fabe4905a9bae5f8909b8f4b 2024-11-09T21:17:47,164 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3756a5c7fabe4905a9bae5f8909b8f4b, entries=250, sequenceid=121, filesize=16.3 K 2024-11-09T21:17:47,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/6b5056496dee43ebbb30a0cf73751f8b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/6b5056496dee43ebbb30a0cf73751f8b 2024-11-09T21:17:47,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/6b5056496dee43ebbb30a0cf73751f8b, entries=150, sequenceid=121, filesize=11.7 K 2024-11-09T21:17:47,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/e38d25a6abdd4311bde8c14e60edb81d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e38d25a6abdd4311bde8c14e60edb81d 2024-11-09T21:17:47,170 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e38d25a6abdd4311bde8c14e60edb81d, entries=150, sequenceid=121, filesize=11.7 K 2024-11-09T21:17:47,170 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 2b19020342a14d63e946c9297523dd88 in 1238ms, sequenceid=121, compaction requested=true 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:47,171 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:47,171 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:47,171 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:47,172 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:47,172 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:47,172 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c20bf16cbf684dcfb2d6c05d66ac4b76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a38726f3a99a4eea8ef27df31f88b5b3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/6b5056496dee43ebbb30a0cf73751f8b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.4 K 2024-11-09T21:17:47,172 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/67aa496083ff4e9fac95c78914290bdf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a05a0bdbe73f4cbd81a301004feaa8f9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3756a5c7fabe4905a9bae5f8909b8f4b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=39.9 K 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67aa496083ff4e9fac95c78914290bdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731187064053 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c20bf16cbf684dcfb2d6c05d66ac4b76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731187064053 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a05a0bdbe73f4cbd81a301004feaa8f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731187064407 2024-11-09T21:17:47,172 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a38726f3a99a4eea8ef27df31f88b5b3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731187064407 2024-11-09T21:17:47,173 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3756a5c7fabe4905a9bae5f8909b8f4b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731187065593 2024-11-09T21:17:47,173 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 6b5056496dee43ebbb30a0cf73751f8b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731187065593 2024-11-09T21:17:47,180 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#397 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:47,180 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#396 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:47,180 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ece1a8ae7d194b248db4ea343b80ed4f is 50, key is test_row_0/B:col10/1731187065932/Put/seqid=0 2024-11-09T21:17:47,180 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/c19c35465914472f83dc7a0bb8c08926 is 50, key is test_row_0/A:col10/1731187065932/Put/seqid=0 2024-11-09T21:17:47,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742295_1471 (size=12309) 2024-11-09T21:17:47,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742294_1470 (size=12309) 2024-11-09T21:17:47,198 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ece1a8ae7d194b248db4ea343b80ed4f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ece1a8ae7d194b248db4ea343b80ed4f 2024-11-09T21:17:47,206 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into ece1a8ae7d194b248db4ea343b80ed4f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:47,206 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:47,206 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187067171; duration=0sec 2024-11-09T21:17:47,206 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:47,206 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:47,206 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:47,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36209 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:47,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:47,207 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:47,207 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8fda6af462a44411820007c99996f709, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9fe2f32525884117bb18e9c3695b58e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e38d25a6abdd4311bde8c14e60edb81d] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.4 K 2024-11-09T21:17:47,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fda6af462a44411820007c99996f709, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731187064053 2024-11-09T21:17:47,207 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fe2f32525884117bb18e9c3695b58e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731187064407 2024-11-09T21:17:47,208 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e38d25a6abdd4311bde8c14e60edb81d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731187065593 2024-11-09T21:17:47,212 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:47,213 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/216713a0def8487ab02cffec1afff9cc is 50, key is test_row_0/C:col10/1731187065932/Put/seqid=0 2024-11-09T21:17:47,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742296_1472 (size=12309) 2024-11-09T21:17:47,275 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-09T21:17:47,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:47,275 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:47,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:47,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/0b87b55c20ca40e797bd830e19f6e884 is 50, key is test_row_0/A:col10/1731187065963/Put/seqid=0 2024-11-09T21:17:47,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742297_1473 (size=12151) 2024-11-09T21:17:47,596 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/c19c35465914472f83dc7a0bb8c08926 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/c19c35465914472f83dc7a0bb8c08926 2024-11-09T21:17:47,610 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into c19c35465914472f83dc7a0bb8c08926(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:47,610 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:47,610 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187067171; duration=0sec 2024-11-09T21:17:47,610 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:47,610 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:47,621 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/216713a0def8487ab02cffec1afff9cc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/216713a0def8487ab02cffec1afff9cc 2024-11-09T21:17:47,624 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into 216713a0def8487ab02cffec1afff9cc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:47,624 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:47,624 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187067171; duration=0sec 2024-11-09T21:17:47,624 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:47,624 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:47,683 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/0b87b55c20ca40e797bd830e19f6e884 2024-11-09T21:17:47,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/d70cbc5f9d034dda8b2e35c58bba4c76 is 50, key is test_row_0/B:col10/1731187065963/Put/seqid=0 2024-11-09T21:17:47,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742298_1474 (size=12151) 2024-11-09T21:17:48,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:48,106 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/d70cbc5f9d034dda8b2e35c58bba4c76 2024-11-09T21:17:48,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/345e4ae51fcd4455b3357b061f4ad525 is 50, key is test_row_0/C:col10/1731187065963/Put/seqid=0 2024-11-09T21:17:48,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742299_1475 (size=12151) 2024-11-09T21:17:48,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:48,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187128159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187128155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,167 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187128159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,167 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187128166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,170 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,177 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187128167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187128276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187128277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,282 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187128277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187128277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,293 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187128286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,293 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187128487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187128487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187128487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187128487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187128496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,500 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,518 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/345e4ae51fcd4455b3357b061f4ad525 2024-11-09T21:17:48,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/0b87b55c20ca40e797bd830e19f6e884 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0b87b55c20ca40e797bd830e19f6e884 2024-11-09T21:17:48,523 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0b87b55c20ca40e797bd830e19f6e884, entries=150, sequenceid=137, filesize=11.9 K 2024-11-09T21:17:48,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/d70cbc5f9d034dda8b2e35c58bba4c76 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/d70cbc5f9d034dda8b2e35c58bba4c76 2024-11-09T21:17:48,526 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/d70cbc5f9d034dda8b2e35c58bba4c76, entries=150, sequenceid=137, filesize=11.9 K 2024-11-09T21:17:48,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/345e4ae51fcd4455b3357b061f4ad525 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/345e4ae51fcd4455b3357b061f4ad525 2024-11-09T21:17:48,529 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/345e4ae51fcd4455b3357b061f4ad525, entries=150, sequenceid=137, filesize=11.9 K 2024-11-09T21:17:48,529 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2b19020342a14d63e946c9297523dd88 in 1254ms, sequenceid=137, compaction requested=false 2024-11-09T21:17:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:48,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-09T21:17:48,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-09T21:17:48,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-09T21:17:48,532 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6320 sec 2024-11-09T21:17:48,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 2.6360 sec 2024-11-09T21:17:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:48,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:17:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:48,801 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:48,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d216fcd662c04f4ea64c04ae8b08c4c1 is 50, key is test_row_0/A:col10/1731187068799/Put/seqid=0 2024-11-09T21:17:48,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742300_1476 (size=14541) 2024-11-09T21:17:48,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d216fcd662c04f4ea64c04ae8b08c4c1 2024-11-09T21:17:48,815 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/646a0437956640d7910884cdc0116150 is 50, key is test_row_0/B:col10/1731187068799/Put/seqid=0 2024-11-09T21:17:48,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742301_1477 (size=12151) 2024-11-09T21:17:48,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/646a0437956640d7910884cdc0116150 2024-11-09T21:17:48,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/6fc49537df0245bdbff719c66d2f5665 is 50, key is test_row_0/C:col10/1731187068799/Put/seqid=0 2024-11-09T21:17:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187128815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187128816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187128818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187128822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,831 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187128831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742302_1478 (size=12151) 2024-11-09T21:17:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187128936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187128936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,945 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187128936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187128936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:48,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:48,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187128946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:48,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187129156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187129156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187129156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,160 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187129156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187129166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/6fc49537df0245bdbff719c66d2f5665 2024-11-09T21:17:49,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d216fcd662c04f4ea64c04ae8b08c4c1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d216fcd662c04f4ea64c04ae8b08c4c1 2024-11-09T21:17:49,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d216fcd662c04f4ea64c04ae8b08c4c1, entries=200, sequenceid=157, filesize=14.2 K 2024-11-09T21:17:49,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/646a0437956640d7910884cdc0116150 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/646a0437956640d7910884cdc0116150 2024-11-09T21:17:49,246 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/646a0437956640d7910884cdc0116150, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T21:17:49,246 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/6fc49537df0245bdbff719c66d2f5665 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/6fc49537df0245bdbff719c66d2f5665 2024-11-09T21:17:49,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/6fc49537df0245bdbff719c66d2f5665, entries=150, sequenceid=157, filesize=11.9 K 2024-11-09T21:17:49,250 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2b19020342a14d63e946c9297523dd88 in 450ms, sequenceid=157, compaction requested=true 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:49,250 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:49,250 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:49,250 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:49,251 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:49,251 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:49,251 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:49,251 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ece1a8ae7d194b248db4ea343b80ed4f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/d70cbc5f9d034dda8b2e35c58bba4c76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/646a0437956640d7910884cdc0116150] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.8 K 2024-11-09T21:17:49,251 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39001 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:49,251 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:49,251 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:49,251 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/c19c35465914472f83dc7a0bb8c08926, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0b87b55c20ca40e797bd830e19f6e884, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d216fcd662c04f4ea64c04ae8b08c4c1] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=38.1 K 2024-11-09T21:17:49,252 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ece1a8ae7d194b248db4ea343b80ed4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731187065593 2024-11-09T21:17:49,252 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting c19c35465914472f83dc7a0bb8c08926, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731187065593 2024-11-09T21:17:49,252 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting d70cbc5f9d034dda8b2e35c58bba4c76, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731187065963 2024-11-09T21:17:49,252 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b87b55c20ca40e797bd830e19f6e884, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731187065963 2024-11-09T21:17:49,252 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 646a0437956640d7910884cdc0116150, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187068157 2024-11-09T21:17:49,252 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d216fcd662c04f4ea64c04ae8b08c4c1, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187068157 2024-11-09T21:17:49,257 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#405 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:49,258 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e656551c9db749eba1e6f87f1ab9639b is 50, key is test_row_0/A:col10/1731187068799/Put/seqid=0 2024-11-09T21:17:49,260 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:49,260 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/b1150649b2194840bbf7fa8f8ffaf6e5 is 50, key is test_row_0/B:col10/1731187068799/Put/seqid=0 2024-11-09T21:17:49,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742303_1479 (size=12561) 2024-11-09T21:17:49,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742304_1480 (size=12561) 2024-11-09T21:17:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:49,470 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:49,470 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:49,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:49,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:49,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:49,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:49,471 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:49,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a3976315c5714202ac25e54dcb78990b is 50, key is test_row_0/A:col10/1731187068815/Put/seqid=0 2024-11-09T21:17:49,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742305_1481 (size=14541) 2024-11-09T21:17:49,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187129490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187129499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187129499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187129500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,504 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187129500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187129606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187129607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187129607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187129607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187129616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,664 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e656551c9db749eba1e6f87f1ab9639b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e656551c9db749eba1e6f87f1ab9639b 2024-11-09T21:17:49,668 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/b1150649b2194840bbf7fa8f8ffaf6e5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b1150649b2194840bbf7fa8f8ffaf6e5 2024-11-09T21:17:49,668 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into e656551c9db749eba1e6f87f1ab9639b(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:49,668 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:49,669 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187069250; duration=0sec 2024-11-09T21:17:49,669 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:49,669 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:49,669 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:49,669 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36611 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:49,669 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:49,669 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:49,669 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/216713a0def8487ab02cffec1afff9cc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/345e4ae51fcd4455b3357b061f4ad525, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/6fc49537df0245bdbff719c66d2f5665] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=35.8 K 2024-11-09T21:17:49,670 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 216713a0def8487ab02cffec1afff9cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=121, earliestPutTs=1731187065593 2024-11-09T21:17:49,670 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 345e4ae51fcd4455b3357b061f4ad525, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731187065963 2024-11-09T21:17:49,670 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fc49537df0245bdbff719c66d2f5665, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187068157 2024-11-09T21:17:49,671 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into b1150649b2194840bbf7fa8f8ffaf6e5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:49,671 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:49,671 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187069250; duration=0sec 2024-11-09T21:17:49,671 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:49,671 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:49,674 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#408 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:49,675 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/5a244fb100ec4043a41f4332a66774ef is 50, key is test_row_0/C:col10/1731187068799/Put/seqid=0 2024-11-09T21:17:49,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742306_1482 (size=12561) 2024-11-09T21:17:49,683 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/5a244fb100ec4043a41f4332a66774ef as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5a244fb100ec4043a41f4332a66774ef 2024-11-09T21:17:49,686 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into 5a244fb100ec4043a41f4332a66774ef(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:49,686 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:49,686 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187069250; duration=0sec 2024-11-09T21:17:49,686 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:49,686 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187129816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187129816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187129816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187129816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187129836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:49,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:49,878 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a3976315c5714202ac25e54dcb78990b 2024-11-09T21:17:49,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e8fe6db73ae74a82b2841a9daf3365f0 is 50, key is test_row_0/B:col10/1731187068815/Put/seqid=0 2024-11-09T21:17:49,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742307_1483 (size=12151) 2024-11-09T21:17:50,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-09T21:17:50,056 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:50,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-09T21:17:50,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:50,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T21:17:50,059 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:50,059 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:50,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187130126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187130127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,132 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187130136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187130136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187130146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T21:17:50,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-09T21:17:50,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,287 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e8fe6db73ae74a82b2841a9daf3365f0 2024-11-09T21:17:50,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8f65add241ad427586c94dd21d0fe961 is 50, key is test_row_0/C:col10/1731187068815/Put/seqid=0 2024-11-09T21:17:50,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742308_1484 (size=12151) 2024-11-09T21:17:50,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-09T21:17:50,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:50,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T21:17:50,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-09T21:17:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187130636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187130636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,642 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187130646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187130656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,668 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-09T21:17:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:50,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:50,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187130666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:50,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:50,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T21:17:50,698 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8f65add241ad427586c94dd21d0fe961 2024-11-09T21:17:50,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a3976315c5714202ac25e54dcb78990b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a3976315c5714202ac25e54dcb78990b 2024-11-09T21:17:50,709 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a3976315c5714202ac25e54dcb78990b, entries=200, sequenceid=172, filesize=14.2 K 2024-11-09T21:17:50,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e8fe6db73ae74a82b2841a9daf3365f0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e8fe6db73ae74a82b2841a9daf3365f0 2024-11-09T21:17:50,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e8fe6db73ae74a82b2841a9daf3365f0, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T21:17:50,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/8f65add241ad427586c94dd21d0fe961 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f65add241ad427586c94dd21d0fe961 2024-11-09T21:17:50,717 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f65add241ad427586c94dd21d0fe961, entries=150, sequenceid=172, filesize=11.9 K 2024-11-09T21:17:50,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2b19020342a14d63e946c9297523dd88 in 1247ms, sequenceid=172, compaction requested=false 2024-11-09T21:17:50,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:50,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-09T21:17:50,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:50,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:50,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:50,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:50,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:50,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:50,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:50,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:50,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e7a5ad1fff0f40b7b66fac9863081d02 is 50, key is test_row_0/A:col10/1731187069489/Put/seqid=0 2024-11-09T21:17:50,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742309_1485 (size=12151) 2024-11-09T21:17:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T21:17:51,229 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e7a5ad1fff0f40b7b66fac9863081d02 2024-11-09T21:17:51,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/f63bd4f94a8f4456973341aceaaf94fc is 50, key is test_row_0/B:col10/1731187069489/Put/seqid=0 2024-11-09T21:17:51,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742310_1486 (size=12151) 2024-11-09T21:17:51,657 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/f63bd4f94a8f4456973341aceaaf94fc 2024-11-09T21:17:51,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:51,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:51,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/278d8106963841878e835dfc0d20451a is 50, key is test_row_0/C:col10/1731187069489/Put/seqid=0 2024-11-09T21:17:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742311_1487 (size=12151) 2024-11-09T21:17:51,670 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/278d8106963841878e835dfc0d20451a 2024-11-09T21:17:51,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e7a5ad1fff0f40b7b66fac9863081d02 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e7a5ad1fff0f40b7b66fac9863081d02 2024-11-09T21:17:51,680 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e7a5ad1fff0f40b7b66fac9863081d02, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T21:17:51,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/f63bd4f94a8f4456973341aceaaf94fc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/f63bd4f94a8f4456973341aceaaf94fc 2024-11-09T21:17:51,684 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/f63bd4f94a8f4456973341aceaaf94fc, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T21:17:51,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/278d8106963841878e835dfc0d20451a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/278d8106963841878e835dfc0d20451a 2024-11-09T21:17:51,687 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/278d8106963841878e835dfc0d20451a, entries=150, sequenceid=192, filesize=11.9 K 2024-11-09T21:17:51,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=6.71 KB/6870 for 2b19020342a14d63e946c9297523dd88 in 872ms, sequenceid=192, compaction requested=true 2024-11-09T21:17:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:51,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-09T21:17:51,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-09T21:17:51,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-09T21:17:51,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6360 sec 2024-11-09T21:17:51,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.6400 sec 2024-11-09T21:17:51,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:51,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:17:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:51,701 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:51,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/f8f91fd8feb744628ee971df472e90ea is 50, key is test_row_0/A:col10/1731187071699/Put/seqid=0 2024-11-09T21:17:51,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742312_1488 (size=14541) 2024-11-09T21:17:51,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/f8f91fd8feb744628ee971df472e90ea 2024-11-09T21:17:51,716 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/8e081b8bd861432492910e393b7f4112 is 50, key is test_row_0/B:col10/1731187071699/Put/seqid=0 2024-11-09T21:17:51,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742313_1489 (size=12151) 2024-11-09T21:17:51,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731187071721, max timestamp=1731187071737Num uniques 859; 2024-11-09T21:17:51,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:17:51,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731187071721, max timestamp=1731187071737Num uniques 859; 2024-11-09T21:17:51,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:17:51,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731187071721, max timestamp=1731187071737Num uniques 859; 2024-11-09T21:17:51,741 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:17:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187131758, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187131770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,780 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187131771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187131772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187131773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,894 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187131886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,895 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187131886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,895 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187131897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187131897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:51,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187131897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:51,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187132096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187132106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/8e081b8bd861432492910e393b7f4112 2024-11-09T21:17:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187132116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187132117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187132117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ed7ee584797245afb43cc0824ae0917b is 50, key is test_row_0/C:col10/1731187071699/Put/seqid=0 2024-11-09T21:17:52,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742314_1490 (size=12151) 2024-11-09T21:17:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-09T21:17:52,216 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:52,217 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-09T21:17:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T21:17:52,218 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:52,219 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:52,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T21:17:52,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-09T21:17:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:52,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:52,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187132417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187132426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,435 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187132426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,436 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187132437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187132437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-09T21:17:52,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,524 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:52,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:52,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:52,530 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ed7ee584797245afb43cc0824ae0917b 2024-11-09T21:17:52,533 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/f8f91fd8feb744628ee971df472e90ea as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/f8f91fd8feb744628ee971df472e90ea 2024-11-09T21:17:52,536 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/f8f91fd8feb744628ee971df472e90ea, entries=200, sequenceid=202, filesize=14.2 K 2024-11-09T21:17:52,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T21:17:52,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/8e081b8bd861432492910e393b7f4112 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8e081b8bd861432492910e393b7f4112 2024-11-09T21:17:52,540 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8e081b8bd861432492910e393b7f4112, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T21:17:52,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ed7ee584797245afb43cc0824ae0917b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ed7ee584797245afb43cc0824ae0917b 2024-11-09T21:17:52,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ed7ee584797245afb43cc0824ae0917b, entries=150, sequenceid=202, filesize=11.9 K 2024-11-09T21:17:52,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 2b19020342a14d63e946c9297523dd88 in 850ms, sequenceid=202, compaction requested=true 2024-11-09T21:17:52,550 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:52,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:52,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:52,550 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:17:52,550 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:17:52,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:52,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:52,550 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:52,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:52,551 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:17:52,551 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:52,551 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:17:52,551 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:52,551 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,552 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,552 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e656551c9db749eba1e6f87f1ab9639b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a3976315c5714202ac25e54dcb78990b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e7a5ad1fff0f40b7b66fac9863081d02, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/f8f91fd8feb744628ee971df472e90ea] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=52.5 K 2024-11-09T21:17:52,552 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b1150649b2194840bbf7fa8f8ffaf6e5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e8fe6db73ae74a82b2841a9daf3365f0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/f63bd4f94a8f4456973341aceaaf94fc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8e081b8bd861432492910e393b7f4112] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=47.9 K 2024-11-09T21:17:52,552 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e656551c9db749eba1e6f87f1ab9639b, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187068157 2024-11-09T21:17:52,552 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b1150649b2194840bbf7fa8f8ffaf6e5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187068157 2024-11-09T21:17:52,552 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3976315c5714202ac25e54dcb78990b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731187068815 2024-11-09T21:17:52,552 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e8fe6db73ae74a82b2841a9daf3365f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731187068815 2024-11-09T21:17:52,552 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e7a5ad1fff0f40b7b66fac9863081d02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731187069489 2024-11-09T21:17:52,553 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f63bd4f94a8f4456973341aceaaf94fc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731187069489 2024-11-09T21:17:52,553 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8f91fd8feb744628ee971df472e90ea, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731187071659 2024-11-09T21:17:52,553 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e081b8bd861432492910e393b7f4112, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731187071659 2024-11-09T21:17:52,563 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#417 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:52,563 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d1a0ae01819b420bb94cea2ac29339d1 is 50, key is test_row_0/A:col10/1731187071699/Put/seqid=0 2024-11-09T21:17:52,566 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:52,567 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/a712b873a1754086b18846d3dfd7783c is 50, key is test_row_0/B:col10/1731187071699/Put/seqid=0 2024-11-09T21:17:52,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742316_1492 (size=12697) 2024-11-09T21:17:52,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742315_1491 (size=12697) 2024-11-09T21:17:52,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-09T21:17:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:52,676 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-09T21:17:52,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:52,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d80fef4f6f8741a0ad21563c48d379c9 is 50, key is test_row_0/A:col10/1731187071772/Put/seqid=0 2024-11-09T21:17:52,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742317_1493 (size=12151) 2024-11-09T21:17:52,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T21:17:52,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:52,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:52,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187132957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,966 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187132958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,966 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187132959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187132965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187132966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:52,982 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/a712b873a1754086b18846d3dfd7783c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a712b873a1754086b18846d3dfd7783c 2024-11-09T21:17:52,993 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d1a0ae01819b420bb94cea2ac29339d1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d1a0ae01819b420bb94cea2ac29339d1 2024-11-09T21:17:52,997 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into a712b873a1754086b18846d3dfd7783c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:52,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:52,997 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=12, startTime=1731187072550; duration=0sec 2024-11-09T21:17:52,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:52,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:52,997 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:17:53,000 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into d1a0ae01819b420bb94cea2ac29339d1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:53,000 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:53,000 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=12, startTime=1731187072550; duration=0sec 2024-11-09T21:17:53,000 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:53,000 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:53,000 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:17:53,000 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:53,000 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,001 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5a244fb100ec4043a41f4332a66774ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f65add241ad427586c94dd21d0fe961, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/278d8106963841878e835dfc0d20451a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ed7ee584797245afb43cc0824ae0917b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=47.9 K 2024-11-09T21:17:53,001 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a244fb100ec4043a41f4332a66774ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731187068157 2024-11-09T21:17:53,001 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f65add241ad427586c94dd21d0fe961, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731187068815 2024-11-09T21:17:53,001 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 278d8106963841878e835dfc0d20451a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731187069489 2024-11-09T21:17:53,001 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ed7ee584797245afb43cc0824ae0917b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731187071659 2024-11-09T21:17:53,009 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#420 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:53,009 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/b8378f5b32f948b4b211b3f3ace502f7 is 50, key is test_row_0/C:col10/1731187071699/Put/seqid=0 2024-11-09T21:17:53,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742318_1494 (size=12697) 2024-11-09T21:17:53,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187133076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187133076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187133076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,083 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,087 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d80fef4f6f8741a0ad21563c48d379c9 2024-11-09T21:17:53,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/13459ec99bbf4717b26e5c26b9aa1d72 is 50, key is test_row_0/B:col10/1731187071772/Put/seqid=0 2024-11-09T21:17:53,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742319_1495 (size=12151) 2024-11-09T21:17:53,097 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/13459ec99bbf4717b26e5c26b9aa1d72 2024-11-09T21:17:53,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187133086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187133086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/fcf544f02e9043f7af5f4b121055acf0 is 50, key is test_row_0/C:col10/1731187071772/Put/seqid=0 2024-11-09T21:17:53,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742320_1496 (size=12151) 2024-11-09T21:17:53,107 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/fcf544f02e9043f7af5f4b121055acf0 2024-11-09T21:17:53,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d80fef4f6f8741a0ad21563c48d379c9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d80fef4f6f8741a0ad21563c48d379c9 2024-11-09T21:17:53,114 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d80fef4f6f8741a0ad21563c48d379c9, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T21:17:53,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/13459ec99bbf4717b26e5c26b9aa1d72 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/13459ec99bbf4717b26e5c26b9aa1d72 2024-11-09T21:17:53,119 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/13459ec99bbf4717b26e5c26b9aa1d72, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T21:17:53,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/fcf544f02e9043f7af5f4b121055acf0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/fcf544f02e9043f7af5f4b121055acf0 2024-11-09T21:17:53,137 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/fcf544f02e9043f7af5f4b121055acf0, entries=150, sequenceid=237, filesize=11.9 K 2024-11-09T21:17:53,138 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 2b19020342a14d63e946c9297523dd88 in 462ms, sequenceid=237, compaction requested=false 2024-11-09T21:17:53,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:53,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-09T21:17:53,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-09T21:17:53,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-09T21:17:53,141 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 920 msec 2024-11-09T21:17:53,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 924 msec 2024-11-09T21:17:53,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:53,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:53,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:53,296 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/afdc22e365514ffeb754dfef4893350b is 50, key is test_row_0/A:col10/1731187072965/Put/seqid=0 2024-11-09T21:17:53,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742321_1497 (size=14541) 2024-11-09T21:17:53,320 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/afdc22e365514ffeb754dfef4893350b 2024-11-09T21:17:53,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e79587b82f3c4caba6c421a154890069 is 50, key is test_row_0/B:col10/1731187072965/Put/seqid=0 2024-11-09T21:17:53,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742322_1498 (size=12151) 2024-11-09T21:17:53,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e79587b82f3c4caba6c421a154890069 2024-11-09T21:17:53,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/a211c12c07994d94bb1d46507a0b943f is 50, key is test_row_0/C:col10/1731187072965/Put/seqid=0 2024-11-09T21:17:53,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742323_1499 (size=12151) 2024-11-09T21:17:53,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187133343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731187073316, max timestamp=1731187073347Num uniques 859; 2024-11-09T21:17:53,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T21:17:53,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731187073316, max timestamp=1731187073347Num uniques 859; 2024-11-09T21:17:53,350 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T21:17:53,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731187073316, max timestamp=1731187073347Num uniques 859; 2024-11-09T21:17:53,351 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-09T21:17:53,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-09T21:17:53,356 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:53,357 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:53,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-09T21:17:53,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T21:17:53,358 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:53,358 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:53,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187133389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187133389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187133397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,405 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,406 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,417 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/b8378f5b32f948b4b211b3f3ace502f7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b8378f5b32f948b4b211b3f3ace502f7 2024-11-09T21:17:53,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187133399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,420 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into b8378f5b32f948b4b211b3f3ace502f7(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:53,420 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:53,420 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=12, startTime=1731187072550; duration=0sec 2024-11-09T21:17:53,420 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:53,420 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T21:17:53,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187133456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,467 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-09T21:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:53,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:53,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:53,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187133507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187133507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187133516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187133526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-09T21:17:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,663 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:53,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:53,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T21:17:53,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187133676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,688 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187133716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187133716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187133736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187133736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:53,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/a211c12c07994d94bb1d46507a0b943f 2024-11-09T21:17:53,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/afdc22e365514ffeb754dfef4893350b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/afdc22e365514ffeb754dfef4893350b 2024-11-09T21:17:53,754 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/afdc22e365514ffeb754dfef4893350b, entries=200, sequenceid=250, filesize=14.2 K 2024-11-09T21:17:53,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e79587b82f3c4caba6c421a154890069 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e79587b82f3c4caba6c421a154890069 2024-11-09T21:17:53,757 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e79587b82f3c4caba6c421a154890069, entries=150, sequenceid=250, filesize=11.9 K 2024-11-09T21:17:53,758 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/a211c12c07994d94bb1d46507a0b943f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a211c12c07994d94bb1d46507a0b943f 2024-11-09T21:17:53,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a211c12c07994d94bb1d46507a0b943f, entries=150, sequenceid=250, filesize=11.9 K 2024-11-09T21:17:53,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 2b19020342a14d63e946c9297523dd88 in 471ms, sequenceid=250, compaction requested=true 2024-11-09T21:17:53,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:53,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:53,761 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:53,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:53,761 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:53,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:53,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:53,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:53,762 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:53,763 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:53,763 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:53,763 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,763 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a712b873a1754086b18846d3dfd7783c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/13459ec99bbf4717b26e5c26b9aa1d72, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e79587b82f3c4caba6c421a154890069] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=36.1 K 2024-11-09T21:17:53,763 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39389 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:53,763 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:53,763 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,763 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d1a0ae01819b420bb94cea2ac29339d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d80fef4f6f8741a0ad21563c48d379c9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/afdc22e365514ffeb754dfef4893350b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=38.5 K 2024-11-09T21:17:53,764 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1a0ae01819b420bb94cea2ac29339d1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731187071659 2024-11-09T21:17:53,764 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a712b873a1754086b18846d3dfd7783c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731187071659 2024-11-09T21:17:53,764 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 13459ec99bbf4717b26e5c26b9aa1d72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187071756 2024-11-09T21:17:53,764 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d80fef4f6f8741a0ad21563c48d379c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187071756 2024-11-09T21:17:53,765 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e79587b82f3c4caba6c421a154890069, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731187072964 2024-11-09T21:17:53,765 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting afdc22e365514ffeb754dfef4893350b, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731187072956 2024-11-09T21:17:53,778 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#426 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:53,779 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a79f910f4ca14d5d9b967f68392f1a51 is 50, key is test_row_0/A:col10/1731187072965/Put/seqid=0 2024-11-09T21:17:53,781 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#427 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:53,781 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ab5826a415f5441383786e089a405aed is 50, key is test_row_0/B:col10/1731187072965/Put/seqid=0 2024-11-09T21:17:53,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742324_1500 (size=12799) 2024-11-09T21:17:53,790 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/a79f910f4ca14d5d9b967f68392f1a51 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a79f910f4ca14d5d9b967f68392f1a51 2024-11-09T21:17:53,795 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into a79f910f4ca14d5d9b967f68392f1a51(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:53,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:53,795 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187073761; duration=0sec 2024-11-09T21:17:53,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:53,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:53,795 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:53,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742325_1501 (size=12799) 2024-11-09T21:17:53,796 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:53,797 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:53,797 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,797 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b8378f5b32f948b4b211b3f3ace502f7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/fcf544f02e9043f7af5f4b121055acf0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a211c12c07994d94bb1d46507a0b943f] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=36.1 K 2024-11-09T21:17:53,798 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8378f5b32f948b4b211b3f3ace502f7, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731187071659 2024-11-09T21:17:53,800 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting fcf544f02e9043f7af5f4b121055acf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731187071756 2024-11-09T21:17:53,800 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a211c12c07994d94bb1d46507a0b943f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731187072964 2024-11-09T21:17:53,807 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/ab5826a415f5441383786e089a405aed as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ab5826a415f5441383786e089a405aed 2024-11-09T21:17:53,808 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:53,809 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/09ecf9aecaa54bc389b1a63d0461038e is 50, key is test_row_0/C:col10/1731187072965/Put/seqid=0 2024-11-09T21:17:53,814 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into ab5826a415f5441383786e089a405aed(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:53,814 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:53,814 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187073761; duration=0sec 2024-11-09T21:17:53,815 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:53,815 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:53,815 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-09T21:17:53,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:53,815 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=134.18 KB heapSize=366.23 KB 2024-11-09T21:17:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:53,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:53,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/8f705c62184948a3adf1b23eab11bfbb is 50, key is test_row_0/A:col10/1731187073366/Put/seqid=0 2024-11-09T21:17:53,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742326_1502 (size=12799) 2024-11-09T21:17:53,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742327_1503 (size=12301) 2024-11-09T21:17:53,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T21:17:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:54,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:54,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187134047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187134048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187134051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,065 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187134057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,066 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187134057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,075 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187134177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,187 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187134177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187134177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,189 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,190 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,227 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/8f705c62184948a3adf1b23eab11bfbb 2024-11-09T21:17:54,230 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/09ecf9aecaa54bc389b1a63d0461038e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/09ecf9aecaa54bc389b1a63d0461038e 2024-11-09T21:17:54,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/3acbfdbb12b04357940130f9e1cf7373 is 50, key is test_row_0/B:col10/1731187073366/Put/seqid=0 2024-11-09T21:17:54,234 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into 09ecf9aecaa54bc389b1a63d0461038e(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:54,235 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:54,235 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187073762; duration=0sec 2024-11-09T21:17:54,235 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:54,235 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:54,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742328_1504 (size=12301) 2024-11-09T21:17:54,242 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/3acbfdbb12b04357940130f9e1cf7373 2024-11-09T21:17:54,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/c137d9ee73ab423cbce1316ad2200f6e is 50, key is test_row_0/C:col10/1731187073366/Put/seqid=0 2024-11-09T21:17:54,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742329_1505 (size=12301) 2024-11-09T21:17:54,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187134397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187134397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187134397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,404 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,405 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T21:17:54,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187134576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,586 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187134587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,593 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,662 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/c137d9ee73ab423cbce1316ad2200f6e 2024-11-09T21:17:54,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/8f705c62184948a3adf1b23eab11bfbb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/8f705c62184948a3adf1b23eab11bfbb 2024-11-09T21:17:54,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/8f705c62184948a3adf1b23eab11bfbb, entries=150, sequenceid=286, filesize=12.0 K 2024-11-09T21:17:54,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/3acbfdbb12b04357940130f9e1cf7373 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/3acbfdbb12b04357940130f9e1cf7373 2024-11-09T21:17:54,672 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/3acbfdbb12b04357940130f9e1cf7373, entries=150, sequenceid=286, filesize=12.0 K 2024-11-09T21:17:54,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/c137d9ee73ab423cbce1316ad2200f6e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/c137d9ee73ab423cbce1316ad2200f6e 2024-11-09T21:17:54,676 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/c137d9ee73ab423cbce1316ad2200f6e, entries=150, sequenceid=286, filesize=12.0 K 2024-11-09T21:17:54,677 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~366.19 KB/374976, currentSize=53.67 KB/54960 for 2b19020342a14d63e946c9297523dd88 in 862ms, sequenceid=286, compaction requested=false 2024-11-09T21:17:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-09T21:17:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-09T21:17:54,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-09T21:17:54,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3210 sec 2024-11-09T21:17:54,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 1.3240 sec 2024-11-09T21:17:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:54,718 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:17:54,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:54,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:54,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:54,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:54,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:54,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:54,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/88f94c0930184433a6a603b034efbd07 is 50, key is test_row_0/A:col10/1731187074043/Put/seqid=0 2024-11-09T21:17:54,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742330_1506 (size=14741) 2024-11-09T21:17:54,765 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/88f94c0930184433a6a603b034efbd07 2024-11-09T21:17:54,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/22ffb370c04d4e6eb4dfb21f792aadbf is 50, key is test_row_0/B:col10/1731187074043/Put/seqid=0 2024-11-09T21:17:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187134778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187134787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187134788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,794 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742331_1507 (size=12301) 2024-11-09T21:17:54,826 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/22ffb370c04d4e6eb4dfb21f792aadbf 2024-11-09T21:17:54,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/28ee5ba309cf4e01bc35b1944cc42fb3 is 50, key is test_row_0/C:col10/1731187074043/Put/seqid=0 2024-11-09T21:17:54,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742332_1508 (size=12301) 2024-11-09T21:17:54,872 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=299 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/28ee5ba309cf4e01bc35b1944cc42fb3 2024-11-09T21:17:54,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/88f94c0930184433a6a603b034efbd07 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/88f94c0930184433a6a603b034efbd07 2024-11-09T21:17:54,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/88f94c0930184433a6a603b034efbd07, entries=200, sequenceid=299, filesize=14.4 K 2024-11-09T21:17:54,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/22ffb370c04d4e6eb4dfb21f792aadbf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/22ffb370c04d4e6eb4dfb21f792aadbf 2024-11-09T21:17:54,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187134897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187134897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/22ffb370c04d4e6eb4dfb21f792aadbf, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T21:17:54,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/28ee5ba309cf4e01bc35b1944cc42fb3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/28ee5ba309cf4e01bc35b1944cc42fb3 2024-11-09T21:17:54,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187134897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:54,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/28ee5ba309cf4e01bc35b1944cc42fb3, entries=150, sequenceid=299, filesize=12.0 K 2024-11-09T21:17:54,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2b19020342a14d63e946c9297523dd88 in 192ms, sequenceid=299, compaction requested=true 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:54,911 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:54,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-09T21:17:54,911 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:54,912 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39841 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:54,912 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:54,912 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:54,912 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a79f910f4ca14d5d9b967f68392f1a51, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/8f705c62184948a3adf1b23eab11bfbb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/88f94c0930184433a6a603b034efbd07] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=38.9 K 2024-11-09T21:17:54,913 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:54,913 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:54,913 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:54,913 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ab5826a415f5441383786e089a405aed, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/3acbfdbb12b04357940130f9e1cf7373, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/22ffb370c04d4e6eb4dfb21f792aadbf] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=36.5 K 2024-11-09T21:17:54,913 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a79f910f4ca14d5d9b967f68392f1a51, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731187072964 2024-11-09T21:17:54,914 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5826a415f5441383786e089a405aed, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731187072964 2024-11-09T21:17:54,914 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f705c62184948a3adf1b23eab11bfbb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731187073366 2024-11-09T21:17:54,914 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3acbfdbb12b04357940130f9e1cf7373, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731187073366 2024-11-09T21:17:54,914 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88f94c0930184433a6a603b034efbd07, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731187074043 2024-11-09T21:17:54,915 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ffb370c04d4e6eb4dfb21f792aadbf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731187074043 2024-11-09T21:17:54,926 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:54,927 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/1a809d41761a4953986b89c80594a6d6 is 50, key is test_row_0/A:col10/1731187074043/Put/seqid=0 2024-11-09T21:17:54,938 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:54,939 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/0812f82fc2a04b70980aa018cd23b767 is 50, key is test_row_0/B:col10/1731187074043/Put/seqid=0 2024-11-09T21:17:54,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742334_1510 (size=13051) 2024-11-09T21:17:54,963 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/0812f82fc2a04b70980aa018cd23b767 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/0812f82fc2a04b70980aa018cd23b767 2024-11-09T21:17:54,967 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into 0812f82fc2a04b70980aa018cd23b767(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:54,967 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:54,967 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187074911; duration=0sec 2024-11-09T21:17:54,968 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:54,968 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:54,968 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:54,969 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:54,969 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:54,969 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:54,969 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/09ecf9aecaa54bc389b1a63d0461038e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/c137d9ee73ab423cbce1316ad2200f6e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/28ee5ba309cf4e01bc35b1944cc42fb3] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=36.5 K 2024-11-09T21:17:54,970 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 09ecf9aecaa54bc389b1a63d0461038e, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731187072964 2024-11-09T21:17:54,973 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting c137d9ee73ab423cbce1316ad2200f6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731187073366 2024-11-09T21:17:54,973 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 28ee5ba309cf4e01bc35b1944cc42fb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731187074043 2024-11-09T21:17:54,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742333_1509 (size=13051) 2024-11-09T21:17:54,988 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#437 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:54,988 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/1ce8782518c048cfb7db14b55216b1d0 is 50, key is test_row_0/C:col10/1731187074043/Put/seqid=0 2024-11-09T21:17:55,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742335_1511 (size=13051) 2024-11-09T21:17:55,037 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/1ce8782518c048cfb7db14b55216b1d0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/1ce8782518c048cfb7db14b55216b1d0 2024-11-09T21:17:55,048 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into 1ce8782518c048cfb7db14b55216b1d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:55,048 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,048 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187074911; duration=0sec 2024-11-09T21:17:55,048 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:55,048 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:55,114 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T21:17:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:55,114 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:17:55,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/14cc687d9d56485dbae19fb424de9e28 is 50, key is test_row_0/A:col10/1731187075113/Put/seqid=0 2024-11-09T21:17:55,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742336_1512 (size=14741) 2024-11-09T21:17:55,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/14cc687d9d56485dbae19fb424de9e28 2024-11-09T21:17:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187135146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187135148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187135155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,164 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,168 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/fd2aecdfd5074bbf9b0fc469676ede7b is 50, key is test_row_0/B:col10/1731187075113/Put/seqid=0 2024-11-09T21:17:55,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742337_1513 (size=12301) 2024-11-09T21:17:55,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/fd2aecdfd5074bbf9b0fc469676ede7b 2024-11-09T21:17:55,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ec28b77d23304b61af6e1aae92c19d55 is 50, key is test_row_0/C:col10/1731187075113/Put/seqid=0 2024-11-09T21:17:55,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742338_1514 (size=12301) 2024-11-09T21:17:55,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=322 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ec28b77d23304b61af6e1aae92c19d55 2024-11-09T21:17:55,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/14cc687d9d56485dbae19fb424de9e28 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/14cc687d9d56485dbae19fb424de9e28 2024-11-09T21:17:55,206 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/14cc687d9d56485dbae19fb424de9e28, entries=200, sequenceid=322, filesize=14.4 K 2024-11-09T21:17:55,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/fd2aecdfd5074bbf9b0fc469676ede7b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/fd2aecdfd5074bbf9b0fc469676ede7b 2024-11-09T21:17:55,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/fd2aecdfd5074bbf9b0fc469676ede7b, entries=150, sequenceid=322, filesize=12.0 K 2024-11-09T21:17:55,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ec28b77d23304b61af6e1aae92c19d55 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ec28b77d23304b61af6e1aae92c19d55 2024-11-09T21:17:55,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ec28b77d23304b61af6e1aae92c19d55, entries=150, sequenceid=322, filesize=12.0 K 2024-11-09T21:17:55,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 2b19020342a14d63e946c9297523dd88 in 105ms, sequenceid=322, compaction requested=false 2024-11-09T21:17:55,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:55,274 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:17:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:55,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:55,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/0267ae20a31d49ed9696d1d31728a3cf is 50, key is test_row_0/A:col10/1731187075273/Put/seqid=0 2024-11-09T21:17:55,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742339_1515 (size=19621) 2024-11-09T21:17:55,300 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/0267ae20a31d49ed9696d1d31728a3cf 2024-11-09T21:17:55,330 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1160961057fa4e8896772145f1ea84d4 is 50, key is test_row_0/B:col10/1731187075273/Put/seqid=0 2024-11-09T21:17:55,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742340_1516 (size=12301) 2024-11-09T21:17:55,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1160961057fa4e8896772145f1ea84d4 2024-11-09T21:17:55,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187135351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,367 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ea2476f9758a4c4db36e732e09c59777 is 50, key is test_row_0/C:col10/1731187075273/Put/seqid=0 2024-11-09T21:17:55,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187135353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742341_1517 (size=12301) 2024-11-09T21:17:55,379 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ea2476f9758a4c4db36e732e09c59777 2024-11-09T21:17:55,381 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187135367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,386 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/0267ae20a31d49ed9696d1d31728a3cf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0267ae20a31d49ed9696d1d31728a3cf 2024-11-09T21:17:55,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0267ae20a31d49ed9696d1d31728a3cf, entries=300, sequenceid=336, filesize=19.2 K 2024-11-09T21:17:55,394 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/1a809d41761a4953986b89c80594a6d6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/1a809d41761a4953986b89c80594a6d6 2024-11-09T21:17:55,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1160961057fa4e8896772145f1ea84d4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1160961057fa4e8896772145f1ea84d4 2024-11-09T21:17:55,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1160961057fa4e8896772145f1ea84d4, entries=150, sequenceid=336, filesize=12.0 K 2024-11-09T21:17:55,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ea2476f9758a4c4db36e732e09c59777 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ea2476f9758a4c4db36e732e09c59777 2024-11-09T21:17:55,400 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into 1a809d41761a4953986b89c80594a6d6(size=12.7 K), total size for store is 46.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:55,400 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,400 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187074911; duration=0sec 2024-11-09T21:17:55,400 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:55,400 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:55,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ea2476f9758a4c4db36e732e09c59777, entries=150, sequenceid=336, filesize=12.0 K 2024-11-09T21:17:55,402 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2b19020342a14d63e946c9297523dd88 in 128ms, sequenceid=336, compaction requested=true 2024-11-09T21:17:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:55,402 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:55,402 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:55,402 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:55,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:55,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:55,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:55,403 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:55,403 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:55,403 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:55,403 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,403 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/0812f82fc2a04b70980aa018cd23b767, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/fd2aecdfd5074bbf9b0fc469676ede7b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1160961057fa4e8896772145f1ea84d4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=36.8 K 2024-11-09T21:17:55,404 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:55,404 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 0812f82fc2a04b70980aa018cd23b767, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731187074043 2024-11-09T21:17:55,404 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:55,404 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,404 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/1a809d41761a4953986b89c80594a6d6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/14cc687d9d56485dbae19fb424de9e28, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0267ae20a31d49ed9696d1d31728a3cf] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=46.3 K 2024-11-09T21:17:55,404 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fd2aecdfd5074bbf9b0fc469676ede7b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731187074727 2024-11-09T21:17:55,404 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1160961057fa4e8896772145f1ea84d4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731187075153 2024-11-09T21:17:55,404 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a809d41761a4953986b89c80594a6d6, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731187074043 2024-11-09T21:17:55,405 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14cc687d9d56485dbae19fb424de9e28, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731187074727 2024-11-09T21:17:55,406 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0267ae20a31d49ed9696d1d31728a3cf, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731187075130 2024-11-09T21:17:55,414 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#444 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:55,415 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/7b22decc7a7f4780bf99acb047f12b98 is 50, key is test_row_0/B:col10/1731187075273/Put/seqid=0 2024-11-09T21:17:55,431 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#445 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:55,431 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3 is 50, key is test_row_0/A:col10/1731187075273/Put/seqid=0 2024-11-09T21:17:55,482 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T21:17:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:55,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742342_1518 (size=13153) 2024-11-09T21:17:55,488 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/7b22decc7a7f4780bf99acb047f12b98 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/7b22decc7a7f4780bf99acb047f12b98 2024-11-09T21:17:55,496 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into 7b22decc7a7f4780bf99acb047f12b98(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:55,496 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,496 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187075402; duration=0sec 2024-11-09T21:17:55,501 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:55,501 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:55,501 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:55,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742343_1519 (size=13153) 2024-11-09T21:17:55,503 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:55,503 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:55,503 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,503 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/1ce8782518c048cfb7db14b55216b1d0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ec28b77d23304b61af6e1aae92c19d55, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ea2476f9758a4c4db36e732e09c59777] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=36.8 K 2024-11-09T21:17:55,504 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce8782518c048cfb7db14b55216b1d0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=299, earliestPutTs=1731187074043 2024-11-09T21:17:55,504 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ec28b77d23304b61af6e1aae92c19d55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=322, earliestPutTs=1731187074727 2024-11-09T21:17:55,505 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ea2476f9758a4c4db36e732e09c59777, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731187075153 2024-11-09T21:17:55,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/4547353d2145466183fc4ee0becf02c4 is 50, key is test_row_0/A:col10/1731187075480/Put/seqid=0 2024-11-09T21:17:55,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-09T21:17:55,517 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:55,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:55,518 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:55,519 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/e89a03c04fc54776a90dda002d71baea is 50, key is test_row_0/C:col10/1731187075273/Put/seqid=0 2024-11-09T21:17:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-09T21:17:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T21:17:55,521 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:55,522 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:55,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:55,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742345_1521 (size=13153) 2024-11-09T21:17:55,549 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187135529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,549 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/e89a03c04fc54776a90dda002d71baea as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e89a03c04fc54776a90dda002d71baea 2024-11-09T21:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,554 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into e89a03c04fc54776a90dda002d71baea(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:55,554 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,554 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187075403; duration=0sec 2024-11-09T21:17:55,554 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:55,554 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:55,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742344_1520 (size=14741) 2024-11-09T21:17:55,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187135546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,566 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187135546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,566 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,568 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/4547353d2145466183fc4ee0becf02c4 2024-11-09T21:17:55,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/9e47a677372e435da1bbe0d5da957eb9 is 50, key is test_row_0/B:col10/1731187075480/Put/seqid=0 2024-11-09T21:17:55,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742346_1522 (size=12301) 2024-11-09T21:17:55,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187135607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,613 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187135607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,614 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 2b19020342a14d63e946c9297523dd88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:55,614 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 2b19020342a14d63e946c9297523dd88 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,614 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T21:17:55,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187135656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-09T21:17:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,684 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187135677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187135677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-09T21:17:55,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:55,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T21:17:55,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187135866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,874 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187135897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:55,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187135897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:55,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:55,907 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3 2024-11-09T21:17:55,910 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into b3e7ef6e72dc4a55b0c3faa2d7e63ba3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:55,910 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:55,910 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187075402; duration=0sec 2024-11-09T21:17:55,911 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:55,911 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:55,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-09T21:17:55,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:55,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:55,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/9e47a677372e435da1bbe0d5da957eb9 2024-11-09T21:17:55,996 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/96b92391fb2d49178545e517640ad2a6 is 50, key is test_row_0/C:col10/1731187075480/Put/seqid=0 2024-11-09T21:17:56,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742347_1523 (size=12301) 2024-11-09T21:17:56,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/96b92391fb2d49178545e517640ad2a6 2024-11-09T21:17:56,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/4547353d2145466183fc4ee0becf02c4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/4547353d2145466183fc4ee0becf02c4 2024-11-09T21:17:56,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/4547353d2145466183fc4ee0becf02c4, entries=200, sequenceid=356, filesize=14.4 K 2024-11-09T21:17:56,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/9e47a677372e435da1bbe0d5da957eb9 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9e47a677372e435da1bbe0d5da957eb9 2024-11-09T21:17:56,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9e47a677372e435da1bbe0d5da957eb9, entries=150, sequenceid=356, filesize=12.0 K 2024-11-09T21:17:56,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/96b92391fb2d49178545e517640ad2a6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/96b92391fb2d49178545e517640ad2a6 2024-11-09T21:17:56,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/96b92391fb2d49178545e517640ad2a6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-09T21:17:56,028 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 2b19020342a14d63e946c9297523dd88 in 546ms, sequenceid=356, compaction requested=false 2024-11-09T21:17:56,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:56,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-09T21:17:56,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:56,133 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T21:17:56,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:56,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:56,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:56,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/43b961e14b754d9587eeb19cd91dd9d2 is 50, key is test_row_0/A:col10/1731187075541/Put/seqid=0 2024-11-09T21:17:56,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742348_1524 (size=12301) 2024-11-09T21:17:56,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T21:17:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:56,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:56,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187136240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,255 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,255 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187136241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187136249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187136366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187136366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,375 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187136366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,376 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,542 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/43b961e14b754d9587eeb19cd91dd9d2 2024-11-09T21:17:56,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e1224a9da4e04d298b4b288e676e8baf is 50, key is test_row_0/B:col10/1731187075541/Put/seqid=0 2024-11-09T21:17:56,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742349_1525 (size=12301) 2024-11-09T21:17:56,555 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e1224a9da4e04d298b4b288e676e8baf 2024-11-09T21:17:56,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/bf7d0c8f8164456e9155a40ae2a38417 is 50, key is test_row_0/C:col10/1731187075541/Put/seqid=0 2024-11-09T21:17:56,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742350_1526 (size=12301) 2024-11-09T21:17:56,572 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/bf7d0c8f8164456e9155a40ae2a38417 2024-11-09T21:17:56,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/43b961e14b754d9587eeb19cd91dd9d2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/43b961e14b754d9587eeb19cd91dd9d2 2024-11-09T21:17:56,588 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/43b961e14b754d9587eeb19cd91dd9d2, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T21:17:56,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/e1224a9da4e04d298b4b288e676e8baf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e1224a9da4e04d298b4b288e676e8baf 2024-11-09T21:17:56,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187136586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187136587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e1224a9da4e04d298b4b288e676e8baf, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T21:17:56,594 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187136587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/bf7d0c8f8164456e9155a40ae2a38417 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/bf7d0c8f8164456e9155a40ae2a38417 2024-11-09T21:17:56,602 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/bf7d0c8f8164456e9155a40ae2a38417, entries=150, sequenceid=373, filesize=12.0 K 2024-11-09T21:17:56,603 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2b19020342a14d63e946c9297523dd88 in 470ms, sequenceid=373, compaction requested=true 2024-11-09T21:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:56,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-09T21:17:56,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-09T21:17:56,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-09T21:17:56,607 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0840 sec 2024-11-09T21:17:56,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 1.0910 sec 2024-11-09T21:17:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-09T21:17:56,657 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:56,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-09T21:17:56,658 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-09T21:17:56,659 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:56,659 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:56,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-09T21:17:56,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:56,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:56,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:56,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/be150f9a30d34138a3a81fef4dbb7564 is 50, key is test_row_0/A:col10/1731187076231/Put/seqid=0 2024-11-09T21:17:56,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742351_1527 (size=12301) 2024-11-09T21:17:56,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:56,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:56,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187136928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187136936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187136938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-09T21:17:57,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187137047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,052 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187137056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,065 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187137056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,219 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/be150f9a30d34138a3a81fef4dbb7564 2024-11-09T21:17:57,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1f4b2da24a174007be03b7285b7c11e0 is 50, key is test_row_0/B:col10/1731187076231/Put/seqid=0 2024-11-09T21:17:57,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742352_1528 (size=12301) 2024-11-09T21:17:57,235 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1f4b2da24a174007be03b7285b7c11e0 2024-11-09T21:17:57,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/5f19b624b8c849f0bab3c7b41531861f is 50, key is test_row_0/C:col10/1731187076231/Put/seqid=0 2024-11-09T21:17:57,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742353_1529 (size=12301) 2024-11-09T21:17:57,255 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=391 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/5f19b624b8c849f0bab3c7b41531861f 2024-11-09T21:17:57,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/be150f9a30d34138a3a81fef4dbb7564 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/be150f9a30d34138a3a81fef4dbb7564 2024-11-09T21:17:57,262 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/be150f9a30d34138a3a81fef4dbb7564, entries=150, sequenceid=391, filesize=12.0 K 2024-11-09T21:17:57,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1f4b2da24a174007be03b7285b7c11e0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1f4b2da24a174007be03b7285b7c11e0 2024-11-09T21:17:57,265 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1f4b2da24a174007be03b7285b7c11e0, entries=150, sequenceid=391, filesize=12.0 K 2024-11-09T21:17:57,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/5f19b624b8c849f0bab3c7b41531861f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5f19b624b8c849f0bab3c7b41531861f 2024-11-09T21:17:57,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187137256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,270 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5f19b624b8c849f0bab3c7b41531861f, entries=150, sequenceid=391, filesize=12.0 K 2024-11-09T21:17:57,271 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2b19020342a14d63e946c9297523dd88 in 460ms, sequenceid=391, compaction requested=true 2024-11-09T21:17:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-09T21:17:57,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-09T21:17:57,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-09T21:17:57,279 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 618 msec 2024-11-09T21:17:57,284 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 625 msec 2024-11-09T21:17:57,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-09T21:17:57,287 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:57,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:57,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-09T21:17:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-09T21:17:57,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-09T21:17:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:57,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:57,289 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:57,290 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:57,291 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:57,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/cbf8beafa1a44fd287be52138f8a2d4b is 50, key is test_row_0/A:col10/1731187077286/Put/seqid=0 2024-11-09T21:17:57,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742354_1530 (size=14741) 2024-11-09T21:17:57,307 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/cbf8beafa1a44fd287be52138f8a2d4b 2024-11-09T21:17:57,320 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/830b54fd89944b5d91574f82e5347e93 is 50, key is test_row_0/B:col10/1731187077286/Put/seqid=0 2024-11-09T21:17:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742355_1531 (size=12301) 2024-11-09T21:17:57,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187137358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187137370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-09T21:17:57,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-09T21:17:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,444 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187137476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187137486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,495 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,582 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187137576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-09T21:17:57,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,597 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-09T21:17:57,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60516 deadline: 1731187137617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,629 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 2b19020342a14d63e946c9297523dd88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60480 deadline: 1731187137636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,644 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 2b19020342a14d63e946c9297523dd88 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:17:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,645 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187137697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:57,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187137706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:57,714 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:57,738 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/830b54fd89944b5d91574f82e5347e93 2024-11-09T21:17:57,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/cec5734b87c2451097029576dc7b539a is 50, key is test_row_0/C:col10/1731187077286/Put/seqid=0 2024-11-09T21:17:57,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742356_1532 (size=12301) 2024-11-09T21:17:57,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-09T21:17:57,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:57,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,752 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-09T21:17:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:57,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:57,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-09T21:17:58,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187138006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,014 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187138026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,036 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-09T21:17:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:58,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:58,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:58,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:58,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187138086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,101 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/cec5734b87c2451097029576dc7b539a 2024-11-09T21:17:58,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/cbf8beafa1a44fd287be52138f8a2d4b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cbf8beafa1a44fd287be52138f8a2d4b 2024-11-09T21:17:58,153 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cbf8beafa1a44fd287be52138f8a2d4b, entries=200, sequenceid=406, filesize=14.4 K 2024-11-09T21:17:58,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/830b54fd89944b5d91574f82e5347e93 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/830b54fd89944b5d91574f82e5347e93 2024-11-09T21:17:58,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/830b54fd89944b5d91574f82e5347e93, entries=150, sequenceid=406, filesize=12.0 K 2024-11-09T21:17:58,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/cec5734b87c2451097029576dc7b539a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/cec5734b87c2451097029576dc7b539a 2024-11-09T21:17:58,161 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/cec5734b87c2451097029576dc7b539a, entries=150, sequenceid=406, filesize=12.0 K 2024-11-09T21:17:58,163 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2b19020342a14d63e946c9297523dd88 in 874ms, sequenceid=406, compaction requested=true 2024-11-09T21:17:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:58,163 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:58,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:58,164 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T21:17:58,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:58,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:58,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:58,164 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T21:17:58,164 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 67237 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T21:17:58,164 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:58,164 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62357 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:58,165 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:58,165 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:58,165 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/7b22decc7a7f4780bf99acb047f12b98, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9e47a677372e435da1bbe0d5da957eb9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e1224a9da4e04d298b4b288e676e8baf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1f4b2da24a174007be03b7285b7c11e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/830b54fd89944b5d91574f82e5347e93] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=60.9 K 2024-11-09T21:17:58,165 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/4547353d2145466183fc4ee0becf02c4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/43b961e14b754d9587eeb19cd91dd9d2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/be150f9a30d34138a3a81fef4dbb7564, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cbf8beafa1a44fd287be52138f8a2d4b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=65.7 K 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3e7ef6e72dc4a55b0c3faa2d7e63ba3, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731187075153 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b22decc7a7f4780bf99acb047f12b98, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731187075153 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4547353d2145466183fc4ee0becf02c4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731187075344 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e47a677372e435da1bbe0d5da957eb9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731187075344 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43b961e14b754d9587eeb19cd91dd9d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731187075525 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e1224a9da4e04d298b4b288e676e8baf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731187075525 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting be150f9a30d34138a3a81fef4dbb7564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731187076230 2024-11-09T21:17:58,165 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f4b2da24a174007be03b7285b7c11e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731187076230 2024-11-09T21:17:58,166 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbf8beafa1a44fd287be52138f8a2d4b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731187076923 2024-11-09T21:17:58,166 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 830b54fd89944b5d91574f82e5347e93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731187076923 2024-11-09T21:17:58,172 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#459 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:58,173 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/b97de837df8e4ecabe16bb217fcd88ba is 50, key is test_row_0/B:col10/1731187077286/Put/seqid=0 2024-11-09T21:17:58,176 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:58,176 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/9cbb0046e8b6467e92aa5d9c78dc5340 is 50, key is test_row_0/A:col10/1731187077286/Put/seqid=0 2024-11-09T21:17:58,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742358_1534 (size=13323) 2024-11-09T21:17:58,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742357_1533 (size=13323) 2024-11-09T21:17:58,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:58,210 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:58,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:58,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/15a1d286a53249d7be677d19c150c881 is 50, key is test_row_0/A:col10/1731187077357/Put/seqid=0 2024-11-09T21:17:58,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742359_1535 (size=12301) 2024-11-09T21:17:58,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-09T21:17:58,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:58,537 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:58,584 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/9cbb0046e8b6467e92aa5d9c78dc5340 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9cbb0046e8b6467e92aa5d9c78dc5340 2024-11-09T21:17:58,591 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into 9cbb0046e8b6467e92aa5d9c78dc5340(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:58,591 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:58,591 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=11, startTime=1731187078163; duration=0sec 2024-11-09T21:17:58,591 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:58,591 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:17:58,591 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-09T21:17:58,593 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62357 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-09T21:17:58,593 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:17:58,593 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:58,593 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e89a03c04fc54776a90dda002d71baea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/96b92391fb2d49178545e517640ad2a6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/bf7d0c8f8164456e9155a40ae2a38417, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5f19b624b8c849f0bab3c7b41531861f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/cec5734b87c2451097029576dc7b539a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=60.9 K 2024-11-09T21:17:58,593 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e89a03c04fc54776a90dda002d71baea, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731187075153 2024-11-09T21:17:58,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b92391fb2d49178545e517640ad2a6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1731187075344 2024-11-09T21:17:58,594 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/b97de837df8e4ecabe16bb217fcd88ba as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b97de837df8e4ecabe16bb217fcd88ba 2024-11-09T21:17:58,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf7d0c8f8164456e9155a40ae2a38417, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=373, earliestPutTs=1731187075525 2024-11-09T21:17:58,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f19b624b8c849f0bab3c7b41531861f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=391, earliestPutTs=1731187076230 2024-11-09T21:17:58,594 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting cec5734b87c2451097029576dc7b539a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731187076923 2024-11-09T21:17:58,597 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into b97de837df8e4ecabe16bb217fcd88ba(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:58,597 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:58,597 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=11, startTime=1731187078164; duration=0sec 2024-11-09T21:17:58,597 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:58,597 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:17:58,606 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#462 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:58,606 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ddc3fb571cd846e692273e5fe4293141 is 50, key is test_row_0/C:col10/1731187077286/Put/seqid=0 2024-11-09T21:17:58,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742360_1536 (size=13323) 2024-11-09T21:17:58,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187138606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,618 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,620 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187138610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,621 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,624 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/15a1d286a53249d7be677d19c150c881 2024-11-09T21:17:58,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/613368ce62ca4ecfaf831cd98e4cbede is 50, key is test_row_0/B:col10/1731187077357/Put/seqid=0 2024-11-09T21:17:58,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742361_1537 (size=12301) 2024-11-09T21:17:58,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187138726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,733 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187138727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,733 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187138941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187138941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:58,951 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:59,017 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/ddc3fb571cd846e692273e5fe4293141 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ddc3fb571cd846e692273e5fe4293141 2024-11-09T21:17:59,025 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into ddc3fb571cd846e692273e5fe4293141(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:17:59,025 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:59,025 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=11, startTime=1731187078164; duration=0sec 2024-11-09T21:17:59,025 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:59,025 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:17:59,039 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/613368ce62ca4ecfaf831cd98e4cbede 2024-11-09T21:17:59,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/00535eb7547642fea9d93992021e609d is 50, key is test_row_0/C:col10/1731187077357/Put/seqid=0 2024-11-09T21:17:59,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742362_1538 (size=12301) 2024-11-09T21:17:59,060 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/00535eb7547642fea9d93992021e609d 2024-11-09T21:17:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/15a1d286a53249d7be677d19c150c881 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/15a1d286a53249d7be677d19c150c881 2024-11-09T21:17:59,071 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/15a1d286a53249d7be677d19c150c881, entries=150, sequenceid=423, filesize=12.0 K 2024-11-09T21:17:59,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/613368ce62ca4ecfaf831cd98e4cbede as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/613368ce62ca4ecfaf831cd98e4cbede 2024-11-09T21:17:59,075 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/613368ce62ca4ecfaf831cd98e4cbede, entries=150, sequenceid=423, filesize=12.0 K 2024-11-09T21:17:59,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/00535eb7547642fea9d93992021e609d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/00535eb7547642fea9d93992021e609d 2024-11-09T21:17:59,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/00535eb7547642fea9d93992021e609d, entries=150, sequenceid=423, filesize=12.0 K 2024-11-09T21:17:59,080 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2b19020342a14d63e946c9297523dd88 in 870ms, sequenceid=423, compaction requested=false 2024-11-09T21:17:59,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:59,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:59,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-09T21:17:59,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-09T21:17:59,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-09T21:17:59,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7900 sec 2024-11-09T21:17:59,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 1.7970 sec 2024-11-09T21:17:59,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:59,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T21:17:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:59,129 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e2f2247093fc465c94de0c1970918be0 is 50, key is test_row_0/A:col10/1731187079123/Put/seqid=0 2024-11-09T21:17:59,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742363_1539 (size=14741) 2024-11-09T21:17:59,154 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e2f2247093fc465c94de0c1970918be0 2024-11-09T21:17:59,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/bd2363222d22418495da88e9300a30ec is 50, key is test_row_0/B:col10/1731187079123/Put/seqid=0 2024-11-09T21:17:59,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742364_1540 (size=12301) 2024-11-09T21:17:59,194 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/bd2363222d22418495da88e9300a30ec 2024-11-09T21:17:59,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/d0d665c60c6443c7a755f57d6c01a81e is 50, key is test_row_0/C:col10/1731187079123/Put/seqid=0 2024-11-09T21:17:59,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742365_1541 (size=12301) 2024-11-09T21:17:59,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:59,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187139246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60500 deadline: 1731187139256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:59,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60466 deadline: 1731187139256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:59,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:59,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187139356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:59,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-09T21:17:59,446 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:17:59,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:17:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees 2024-11-09T21:17:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-09T21:17:59,449 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:17:59,449 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=147, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:17:59,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:17:59,468 INFO [Thread-1998 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1998. 2024-11-09T21:17:59,468 DEBUG [Thread-1998 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,468 INFO [Thread-2002 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2002. 2024-11-09T21:17:59,468 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,468 DEBUG [Thread-2002 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,468 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,468 DEBUG [Thread-1998 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,468 DEBUG [Thread-2002 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,468 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,474 INFO [Thread-2000 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2000. 2024-11-09T21:17:59,474 DEBUG [Thread-2000 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,474 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,474 DEBUG [Thread-2000 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,474 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,482 INFO [Thread-2006 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2006. 2024-11-09T21:17:59,482 DEBUG [Thread-2006 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,482 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,482 DEBUG [Thread-2006 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,482 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,482 INFO [Thread-2004 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2004. 2024-11-09T21:17:59,482 DEBUG [Thread-2004 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,482 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,482 DEBUG [Thread-2004 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-09T21:17:59,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:17:59,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60496 deadline: 1731187139576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:17:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 , the old value is region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:17:59,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., hostname=9514d6d5e5be,38413,1731186934857, seqNum=2 because the exception is null or not the one we care about 2024-11-09T21:17:59,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T21:17:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:59,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:59,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] handler.RSProcedureHandler(58): pid=148 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:59,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=148 java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:59,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=148 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:17:59,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/d0d665c60c6443c7a755f57d6c01a81e 2024-11-09T21:17:59,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/e2f2247093fc465c94de0c1970918be0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e2f2247093fc465c94de0c1970918be0 2024-11-09T21:17:59,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e2f2247093fc465c94de0c1970918be0, entries=200, sequenceid=442, filesize=14.4 K 2024-11-09T21:17:59,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/bd2363222d22418495da88e9300a30ec as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/bd2363222d22418495da88e9300a30ec 2024-11-09T21:17:59,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/bd2363222d22418495da88e9300a30ec, entries=150, sequenceid=442, filesize=12.0 K 2024-11-09T21:17:59,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/d0d665c60c6443c7a755f57d6c01a81e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/d0d665c60c6443c7a755f57d6c01a81e 2024-11-09T21:17:59,637 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/d0d665c60c6443c7a755f57d6c01a81e, entries=150, sequenceid=442, filesize=12.0 K 2024-11-09T21:17:59,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2b19020342a14d63e946c9297523dd88 in 513ms, sequenceid=442, compaction requested=true 2024-11-09T21:17:59,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:17:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:17:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:17:59,638 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:17:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2b19020342a14d63e946c9297523dd88:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:17:59,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:17:59,638 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:17:59,638 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:59,638 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40365 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:17:59,638 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/B is initiating minor compaction (all files) 2024-11-09T21:17:59,638 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/A is initiating minor compaction (all files) 2024-11-09T21:17:59,638 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/B in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:59,638 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/A in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:59,638 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b97de837df8e4ecabe16bb217fcd88ba, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/613368ce62ca4ecfaf831cd98e4cbede, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/bd2363222d22418495da88e9300a30ec] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=37.0 K 2024-11-09T21:17:59,638 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9cbb0046e8b6467e92aa5d9c78dc5340, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/15a1d286a53249d7be677d19c150c881, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e2f2247093fc465c94de0c1970918be0] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=39.4 K 2024-11-09T21:17:59,639 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting b97de837df8e4ecabe16bb217fcd88ba, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731187076923 2024-11-09T21:17:59,639 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9cbb0046e8b6467e92aa5d9c78dc5340, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731187076923 2024-11-09T21:17:59,639 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 613368ce62ca4ecfaf831cd98e4cbede, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731187077321 2024-11-09T21:17:59,639 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15a1d286a53249d7be677d19c150c881, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731187077321 2024-11-09T21:17:59,639 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bd2363222d22418495da88e9300a30ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731187078602 2024-11-09T21:17:59,639 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2f2247093fc465c94de0c1970918be0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731187078586 2024-11-09T21:17:59,644 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#B#compaction#468 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:59,644 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#A#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:17:59,645 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/2ef74888978f40c3b790f59a24a8838d is 50, key is test_row_0/A:col10/1731187079123/Put/seqid=0 2024-11-09T21:17:59,645 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/970132cb394a452eb170996bd37f200b is 50, key is test_row_0/B:col10/1731187079123/Put/seqid=0 2024-11-09T21:17:59,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742367_1543 (size=13425) 2024-11-09T21:17:59,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742366_1542 (size=13425) 2024-11-09T21:17:59,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=148 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:17:59,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:17:59,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:17:59,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d925034044154c9e9e4107c6311d4a93 is 50, key is test_row_0/A:col10/1731187079210/Put/seqid=0 2024-11-09T21:17:59,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742368_1544 (size=12301) 2024-11-09T21:17:59,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-09T21:17:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 2b19020342a14d63e946c9297523dd88 2024-11-09T21:17:59,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. as already flushing 2024-11-09T21:17:59,778 INFO [Thread-1989 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1989. 2024-11-09T21:17:59,778 INFO [Thread-1993 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1993. 2024-11-09T21:17:59,778 DEBUG [Thread-1993 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,778 DEBUG [Thread-1989 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,778 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,778 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,779 DEBUG [Thread-1989 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,779 DEBUG [Thread-1993 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,779 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:17:59,887 INFO [Thread-1987 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1987. 2024-11-09T21:17:59,887 DEBUG [Thread-1987 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:17:59,887 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,887 DEBUG [Thread-1987 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:17:59,887 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:00,054 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/2ef74888978f40c3b790f59a24a8838d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/2ef74888978f40c3b790f59a24a8838d 2024-11-09T21:18:00,056 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/970132cb394a452eb170996bd37f200b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/970132cb394a452eb170996bd37f200b 2024-11-09T21:18:00,057 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/A of 2b19020342a14d63e946c9297523dd88 into 2ef74888978f40c3b790f59a24a8838d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:00,057 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:18:00,057 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/A, priority=13, startTime=1731187079637; duration=0sec 2024-11-09T21:18:00,058 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:00,058 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:A 2024-11-09T21:18:00,058 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:00,058 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:00,058 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 2b19020342a14d63e946c9297523dd88/C is initiating minor compaction (all files) 2024-11-09T21:18:00,058 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2b19020342a14d63e946c9297523dd88/C in TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:18:00,058 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ddc3fb571cd846e692273e5fe4293141, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/00535eb7547642fea9d93992021e609d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/d0d665c60c6443c7a755f57d6c01a81e] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp, totalSize=37.0 K 2024-11-09T21:18:00,059 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc3fb571cd846e692273e5fe4293141, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=406, earliestPutTs=1731187076923 2024-11-09T21:18:00,059 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00535eb7547642fea9d93992021e609d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731187077321 2024-11-09T21:18:00,059 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0d665c60c6443c7a755f57d6c01a81e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=442, earliestPutTs=1731187078602 2024-11-09T21:18:00,059 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/B of 2b19020342a14d63e946c9297523dd88 into 970132cb394a452eb170996bd37f200b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:00,059 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:18:00,059 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/B, priority=13, startTime=1731187079638; duration=0sec 2024-11-09T21:18:00,059 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:00,059 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:B 2024-11-09T21:18:00,063 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2b19020342a14d63e946c9297523dd88#C#compaction#471 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:00,064 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/361ff29e17214a278e0bb7b871dfde17 is 50, key is test_row_0/C:col10/1731187079123/Put/seqid=0 2024-11-09T21:18:00,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742369_1545 (size=13425) 2024-11-09T21:18:00,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-09T21:18:00,161 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d925034044154c9e9e4107c6311d4a93 2024-11-09T21:18:00,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/8947feb1db984c24bc34fa1d758b0196 is 50, key is test_row_0/B:col10/1731187079210/Put/seqid=0 2024-11-09T21:18:00,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742370_1546 (size=12301) 2024-11-09T21:18:00,470 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/361ff29e17214a278e0bb7b871dfde17 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/361ff29e17214a278e0bb7b871dfde17 2024-11-09T21:18:00,474 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2b19020342a14d63e946c9297523dd88/C of 2b19020342a14d63e946c9297523dd88 into 361ff29e17214a278e0bb7b871dfde17(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:00,474 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:18:00,474 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., storeName=2b19020342a14d63e946c9297523dd88/C, priority=13, startTime=1731187079638; duration=0sec 2024-11-09T21:18:00,474 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:00,474 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2b19020342a14d63e946c9297523dd88:C 2024-11-09T21:18:00,569 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/8947feb1db984c24bc34fa1d758b0196 2024-11-09T21:18:00,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/a202e6bbe5594275b00e44565a75f451 is 50, key is test_row_0/C:col10/1731187079210/Put/seqid=0 2024-11-09T21:18:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742371_1547 (size=12301) 2024-11-09T21:18:00,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-09T21:18:00,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/a202e6bbe5594275b00e44565a75f451 2024-11-09T21:18:00,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/d925034044154c9e9e4107c6311d4a93 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d925034044154c9e9e4107c6311d4a93 2024-11-09T21:18:00,984 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d925034044154c9e9e4107c6311d4a93, entries=150, sequenceid=458, filesize=12.0 K 2024-11-09T21:18:00,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/8947feb1db984c24bc34fa1d758b0196 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8947feb1db984c24bc34fa1d758b0196 2024-11-09T21:18:00,987 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8947feb1db984c24bc34fa1d758b0196, entries=150, sequenceid=458, filesize=12.0 K 2024-11-09T21:18:00,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/a202e6bbe5594275b00e44565a75f451 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a202e6bbe5594275b00e44565a75f451 2024-11-09T21:18:00,991 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a202e6bbe5594275b00e44565a75f451, entries=150, sequenceid=458, filesize=12.0 K 2024-11-09T21:18:00,991 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=20.13 KB/20610 for 2b19020342a14d63e946c9297523dd88 in 1237ms, sequenceid=458, compaction requested=false 2024-11-09T21:18:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.HRegion(2603): Flush status journal for 2b19020342a14d63e946c9297523dd88: 2024-11-09T21:18:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:18:00,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=148}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=148 2024-11-09T21:18:00,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=148 2024-11-09T21:18:00,994 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-09T21:18:00,995 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5440 sec 2024-11-09T21:18:00,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=147, table=TestAcidGuarantees in 1.5480 sec 2024-11-09T21:18:01,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-09T21:18:01,596 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:01,658 INFO [Thread-1991 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1991. 2024-11-09T21:18:01,658 DEBUG [Thread-1991 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:01,658 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:01,658 DEBUG [Thread-1991 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:01,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:01,667 INFO [Thread-1995 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1995. 2024-11-09T21:18:01,667 DEBUG [Thread-1995 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:01,667 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:01,667 DEBUG [Thread-1995 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:01,667 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 94 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1120 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3359 rows 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1118 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3354 rows 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1121 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3363 rows 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1117 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3351 rows 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1118 2024-11-09T21:18:01,667 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3354 rows 2024-11-09T21:18:01,667 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:18:01,668 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:18:01,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:01,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:01,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:01,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T21:18:01,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:01,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-09T21:18:01,672 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187081671"}]},"ts":"1731187081671"} 2024-11-09T21:18:01,673 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T21:18:01,673 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T21:18:01,673 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:18:01,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, UNASSIGN}] 2024-11-09T21:18:01,675 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, UNASSIGN 2024-11-09T21:18:01,675 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=2b19020342a14d63e946c9297523dd88, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:01,676 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=150, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, UNASSIGN because future has completed 2024-11-09T21:18:01,677 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:18:01,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:18:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-09T21:18:01,829 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(122): Close 2b19020342a14d63e946c9297523dd88 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1722): Closing 2b19020342a14d63e946c9297523dd88, disabling compactions & flushes 2024-11-09T21:18:01,829 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. after waiting 0 ms 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:18:01,829 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(2902): Flushing 2b19020342a14d63e946c9297523dd88 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=A 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=B 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2b19020342a14d63e946c9297523dd88, store=C 2024-11-09T21:18:01,829 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:01,832 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/5e795612f8b74462a90cd3ef88962234 is 50, key is test_row_0/A:col10/1731187079886/Put/seqid=0 2024-11-09T21:18:01,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742372_1548 (size=12301) 2024-11-09T21:18:01,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-09T21:18:02,236 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/5e795612f8b74462a90cd3ef88962234 2024-11-09T21:18:02,241 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1950959ca03841eebffd0844c81ab7cb is 50, key is test_row_0/B:col10/1731187079886/Put/seqid=0 2024-11-09T21:18:02,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742373_1549 (size=12301) 2024-11-09T21:18:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-09T21:18:02,644 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1950959ca03841eebffd0844c81ab7cb 2024-11-09T21:18:02,650 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/3ec9f48e6dba41af951937459d5c2600 is 50, key is test_row_0/C:col10/1731187079886/Put/seqid=0 2024-11-09T21:18:02,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742374_1550 (size=12301) 2024-11-09T21:18:02,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-09T21:18:03,053 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=469 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/3ec9f48e6dba41af951937459d5c2600 2024-11-09T21:18:03,056 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/A/5e795612f8b74462a90cd3ef88962234 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/5e795612f8b74462a90cd3ef88962234 2024-11-09T21:18:03,058 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/5e795612f8b74462a90cd3ef88962234, entries=150, sequenceid=469, filesize=12.0 K 2024-11-09T21:18:03,059 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/B/1950959ca03841eebffd0844c81ab7cb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1950959ca03841eebffd0844c81ab7cb 2024-11-09T21:18:03,061 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1950959ca03841eebffd0844c81ab7cb, entries=150, sequenceid=469, filesize=12.0 K 2024-11-09T21:18:03,061 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/.tmp/C/3ec9f48e6dba41af951937459d5c2600 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/3ec9f48e6dba41af951937459d5c2600 2024-11-09T21:18:03,063 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/3ec9f48e6dba41af951937459d5c2600, entries=150, sequenceid=469, filesize=12.0 K 2024-11-09T21:18:03,064 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2b19020342a14d63e946c9297523dd88 in 1235ms, sequenceid=469, compaction requested=true 2024-11-09T21:18:03,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3d57fbbfd47a4d41af43e3ee4c49a2ae, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/6065ab717b2b47f9b4024666e8fda281, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/ce5fcf9aba1f4b8f859832e1b943d5df, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9e5bb3827c064e0fa85dbc81b12cfa55, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/39e25a524d8046b2bf5eab0ca98b855c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cb4dbec5ede840dab0cfe8d1497853ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/67aa496083ff4e9fac95c78914290bdf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a05a0bdbe73f4cbd81a301004feaa8f9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3756a5c7fabe4905a9bae5f8909b8f4b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/c19c35465914472f83dc7a0bb8c08926, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0b87b55c20ca40e797bd830e19f6e884, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d216fcd662c04f4ea64c04ae8b08c4c1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e656551c9db749eba1e6f87f1ab9639b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a3976315c5714202ac25e54dcb78990b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e7a5ad1fff0f40b7b66fac9863081d02, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/f8f91fd8feb744628ee971df472e90ea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d1a0ae01819b420bb94cea2ac29339d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d80fef4f6f8741a0ad21563c48d379c9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/afdc22e365514ffeb754dfef4893350b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a79f910f4ca14d5d9b967f68392f1a51, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/8f705c62184948a3adf1b23eab11bfbb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/88f94c0930184433a6a603b034efbd07, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/1a809d41761a4953986b89c80594a6d6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/14cc687d9d56485dbae19fb424de9e28, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0267ae20a31d49ed9696d1d31728a3cf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/4547353d2145466183fc4ee0becf02c4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/43b961e14b754d9587eeb19cd91dd9d2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/be150f9a30d34138a3a81fef4dbb7564, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cbf8beafa1a44fd287be52138f8a2d4b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9cbb0046e8b6467e92aa5d9c78dc5340, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/15a1d286a53249d7be677d19c150c881, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e2f2247093fc465c94de0c1970918be0] to archive 2024-11-09T21:18:03,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:18:03,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3d57fbbfd47a4d41af43e3ee4c49a2ae to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3d57fbbfd47a4d41af43e3ee4c49a2ae 2024-11-09T21:18:03,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/6065ab717b2b47f9b4024666e8fda281 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/6065ab717b2b47f9b4024666e8fda281 2024-11-09T21:18:03,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/ce5fcf9aba1f4b8f859832e1b943d5df to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/ce5fcf9aba1f4b8f859832e1b943d5df 2024-11-09T21:18:03,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9e5bb3827c064e0fa85dbc81b12cfa55 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9e5bb3827c064e0fa85dbc81b12cfa55 2024-11-09T21:18:03,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/39e25a524d8046b2bf5eab0ca98b855c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/39e25a524d8046b2bf5eab0ca98b855c 2024-11-09T21:18:03,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cb4dbec5ede840dab0cfe8d1497853ef to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cb4dbec5ede840dab0cfe8d1497853ef 2024-11-09T21:18:03,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/67aa496083ff4e9fac95c78914290bdf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/67aa496083ff4e9fac95c78914290bdf 2024-11-09T21:18:03,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a05a0bdbe73f4cbd81a301004feaa8f9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a05a0bdbe73f4cbd81a301004feaa8f9 2024-11-09T21:18:03,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3756a5c7fabe4905a9bae5f8909b8f4b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/3756a5c7fabe4905a9bae5f8909b8f4b 2024-11-09T21:18:03,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/c19c35465914472f83dc7a0bb8c08926 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/c19c35465914472f83dc7a0bb8c08926 2024-11-09T21:18:03,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0b87b55c20ca40e797bd830e19f6e884 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0b87b55c20ca40e797bd830e19f6e884 2024-11-09T21:18:03,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d216fcd662c04f4ea64c04ae8b08c4c1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d216fcd662c04f4ea64c04ae8b08c4c1 2024-11-09T21:18:03,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e656551c9db749eba1e6f87f1ab9639b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e656551c9db749eba1e6f87f1ab9639b 2024-11-09T21:18:03,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a3976315c5714202ac25e54dcb78990b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a3976315c5714202ac25e54dcb78990b 2024-11-09T21:18:03,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e7a5ad1fff0f40b7b66fac9863081d02 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e7a5ad1fff0f40b7b66fac9863081d02 2024-11-09T21:18:03,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/f8f91fd8feb744628ee971df472e90ea to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/f8f91fd8feb744628ee971df472e90ea 2024-11-09T21:18:03,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d1a0ae01819b420bb94cea2ac29339d1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d1a0ae01819b420bb94cea2ac29339d1 2024-11-09T21:18:03,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d80fef4f6f8741a0ad21563c48d379c9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d80fef4f6f8741a0ad21563c48d379c9 2024-11-09T21:18:03,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/afdc22e365514ffeb754dfef4893350b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/afdc22e365514ffeb754dfef4893350b 2024-11-09T21:18:03,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a79f910f4ca14d5d9b967f68392f1a51 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/a79f910f4ca14d5d9b967f68392f1a51 2024-11-09T21:18:03,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/8f705c62184948a3adf1b23eab11bfbb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/8f705c62184948a3adf1b23eab11bfbb 2024-11-09T21:18:03,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/88f94c0930184433a6a603b034efbd07 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/88f94c0930184433a6a603b034efbd07 2024-11-09T21:18:03,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/1a809d41761a4953986b89c80594a6d6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/1a809d41761a4953986b89c80594a6d6 2024-11-09T21:18:03,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/14cc687d9d56485dbae19fb424de9e28 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/14cc687d9d56485dbae19fb424de9e28 2024-11-09T21:18:03,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0267ae20a31d49ed9696d1d31728a3cf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/0267ae20a31d49ed9696d1d31728a3cf 2024-11-09T21:18:03,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/b3e7ef6e72dc4a55b0c3faa2d7e63ba3 2024-11-09T21:18:03,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/4547353d2145466183fc4ee0becf02c4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/4547353d2145466183fc4ee0becf02c4 2024-11-09T21:18:03,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/43b961e14b754d9587eeb19cd91dd9d2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/43b961e14b754d9587eeb19cd91dd9d2 2024-11-09T21:18:03,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/be150f9a30d34138a3a81fef4dbb7564 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/be150f9a30d34138a3a81fef4dbb7564 2024-11-09T21:18:03,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cbf8beafa1a44fd287be52138f8a2d4b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/cbf8beafa1a44fd287be52138f8a2d4b 2024-11-09T21:18:03,087 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9cbb0046e8b6467e92aa5d9c78dc5340 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/9cbb0046e8b6467e92aa5d9c78dc5340 2024-11-09T21:18:03,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/15a1d286a53249d7be677d19c150c881 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/15a1d286a53249d7be677d19c150c881 2024-11-09T21:18:03,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e2f2247093fc465c94de0c1970918be0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/e2f2247093fc465c94de0c1970918be0 2024-11-09T21:18:03,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/db61fe608902470ebac9fff5f0f6669f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c9ebe5b570e24b07a83a795e73a6efab, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/072e320d602e4843b24efc28450e0053, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ed471fb79226427bbcd24c6123d46358, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/98806f82e69f4b7ca849e3ee2cda6050, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c20bf16cbf684dcfb2d6c05d66ac4b76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9bd9acead8344dcd892a7d94b41bd523, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a38726f3a99a4eea8ef27df31f88b5b3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ece1a8ae7d194b248db4ea343b80ed4f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/6b5056496dee43ebbb30a0cf73751f8b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/d70cbc5f9d034dda8b2e35c58bba4c76, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b1150649b2194840bbf7fa8f8ffaf6e5, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/646a0437956640d7910884cdc0116150, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e8fe6db73ae74a82b2841a9daf3365f0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/f63bd4f94a8f4456973341aceaaf94fc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a712b873a1754086b18846d3dfd7783c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8e081b8bd861432492910e393b7f4112, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/13459ec99bbf4717b26e5c26b9aa1d72, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ab5826a415f5441383786e089a405aed, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e79587b82f3c4caba6c421a154890069, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/3acbfdbb12b04357940130f9e1cf7373, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/0812f82fc2a04b70980aa018cd23b767, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/22ffb370c04d4e6eb4dfb21f792aadbf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/fd2aecdfd5074bbf9b0fc469676ede7b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/7b22decc7a7f4780bf99acb047f12b98, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1160961057fa4e8896772145f1ea84d4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9e47a677372e435da1bbe0d5da957eb9, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e1224a9da4e04d298b4b288e676e8baf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1f4b2da24a174007be03b7285b7c11e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b97de837df8e4ecabe16bb217fcd88ba, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/830b54fd89944b5d91574f82e5347e93, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/613368ce62ca4ecfaf831cd98e4cbede, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/bd2363222d22418495da88e9300a30ec] to archive 2024-11-09T21:18:03,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:18:03,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/db61fe608902470ebac9fff5f0f6669f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/db61fe608902470ebac9fff5f0f6669f 2024-11-09T21:18:03,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c9ebe5b570e24b07a83a795e73a6efab to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c9ebe5b570e24b07a83a795e73a6efab 2024-11-09T21:18:03,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/072e320d602e4843b24efc28450e0053 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/072e320d602e4843b24efc28450e0053 2024-11-09T21:18:03,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ed471fb79226427bbcd24c6123d46358 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ed471fb79226427bbcd24c6123d46358 2024-11-09T21:18:03,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/98806f82e69f4b7ca849e3ee2cda6050 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/98806f82e69f4b7ca849e3ee2cda6050 2024-11-09T21:18:03,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c20bf16cbf684dcfb2d6c05d66ac4b76 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/c20bf16cbf684dcfb2d6c05d66ac4b76 2024-11-09T21:18:03,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9bd9acead8344dcd892a7d94b41bd523 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9bd9acead8344dcd892a7d94b41bd523 2024-11-09T21:18:03,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a38726f3a99a4eea8ef27df31f88b5b3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a38726f3a99a4eea8ef27df31f88b5b3 2024-11-09T21:18:03,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ece1a8ae7d194b248db4ea343b80ed4f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ece1a8ae7d194b248db4ea343b80ed4f 2024-11-09T21:18:03,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/6b5056496dee43ebbb30a0cf73751f8b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/6b5056496dee43ebbb30a0cf73751f8b 2024-11-09T21:18:03,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/d70cbc5f9d034dda8b2e35c58bba4c76 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/d70cbc5f9d034dda8b2e35c58bba4c76 2024-11-09T21:18:03,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b1150649b2194840bbf7fa8f8ffaf6e5 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b1150649b2194840bbf7fa8f8ffaf6e5 2024-11-09T21:18:03,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/646a0437956640d7910884cdc0116150 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/646a0437956640d7910884cdc0116150 2024-11-09T21:18:03,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e8fe6db73ae74a82b2841a9daf3365f0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e8fe6db73ae74a82b2841a9daf3365f0 2024-11-09T21:18:03,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/f63bd4f94a8f4456973341aceaaf94fc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/f63bd4f94a8f4456973341aceaaf94fc 2024-11-09T21:18:03,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a712b873a1754086b18846d3dfd7783c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/a712b873a1754086b18846d3dfd7783c 2024-11-09T21:18:03,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8e081b8bd861432492910e393b7f4112 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8e081b8bd861432492910e393b7f4112 2024-11-09T21:18:03,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/13459ec99bbf4717b26e5c26b9aa1d72 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/13459ec99bbf4717b26e5c26b9aa1d72 2024-11-09T21:18:03,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ab5826a415f5441383786e089a405aed to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/ab5826a415f5441383786e089a405aed 2024-11-09T21:18:03,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e79587b82f3c4caba6c421a154890069 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e79587b82f3c4caba6c421a154890069 2024-11-09T21:18:03,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/3acbfdbb12b04357940130f9e1cf7373 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/3acbfdbb12b04357940130f9e1cf7373 2024-11-09T21:18:03,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/0812f82fc2a04b70980aa018cd23b767 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/0812f82fc2a04b70980aa018cd23b767 2024-11-09T21:18:03,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/22ffb370c04d4e6eb4dfb21f792aadbf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/22ffb370c04d4e6eb4dfb21f792aadbf 2024-11-09T21:18:03,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/fd2aecdfd5074bbf9b0fc469676ede7b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/fd2aecdfd5074bbf9b0fc469676ede7b 2024-11-09T21:18:03,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/7b22decc7a7f4780bf99acb047f12b98 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/7b22decc7a7f4780bf99acb047f12b98 2024-11-09T21:18:03,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1160961057fa4e8896772145f1ea84d4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1160961057fa4e8896772145f1ea84d4 2024-11-09T21:18:03,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9e47a677372e435da1bbe0d5da957eb9 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/9e47a677372e435da1bbe0d5da957eb9 2024-11-09T21:18:03,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e1224a9da4e04d298b4b288e676e8baf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/e1224a9da4e04d298b4b288e676e8baf 2024-11-09T21:18:03,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1f4b2da24a174007be03b7285b7c11e0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1f4b2da24a174007be03b7285b7c11e0 2024-11-09T21:18:03,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b97de837df8e4ecabe16bb217fcd88ba to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/b97de837df8e4ecabe16bb217fcd88ba 2024-11-09T21:18:03,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/830b54fd89944b5d91574f82e5347e93 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/830b54fd89944b5d91574f82e5347e93 2024-11-09T21:18:03,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/613368ce62ca4ecfaf831cd98e4cbede to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/613368ce62ca4ecfaf831cd98e4cbede 2024-11-09T21:18:03,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/bd2363222d22418495da88e9300a30ec to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/bd2363222d22418495da88e9300a30ec 2024-11-09T21:18:03,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f0e07846f13469e81874e96b6d776d1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9dad551c1dce4f8eb6990829c915ffe1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b118933bb9f7452194451e7d2fa20e03, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/98474b11e4374f92acdeae72f6cf93a8, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/76b41ded0c1649d18a2ed94c92f6b74c, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8fda6af462a44411820007c99996f709, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/100d6af7f3ff4b279996f4d10c4cb907, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9fe2f32525884117bb18e9c3695b58e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/216713a0def8487ab02cffec1afff9cc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e38d25a6abdd4311bde8c14e60edb81d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/345e4ae51fcd4455b3357b061f4ad525, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5a244fb100ec4043a41f4332a66774ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/6fc49537df0245bdbff719c66d2f5665, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f65add241ad427586c94dd21d0fe961, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/278d8106963841878e835dfc0d20451a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b8378f5b32f948b4b211b3f3ace502f7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ed7ee584797245afb43cc0824ae0917b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/fcf544f02e9043f7af5f4b121055acf0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/09ecf9aecaa54bc389b1a63d0461038e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a211c12c07994d94bb1d46507a0b943f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/c137d9ee73ab423cbce1316ad2200f6e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/1ce8782518c048cfb7db14b55216b1d0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/28ee5ba309cf4e01bc35b1944cc42fb3, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ec28b77d23304b61af6e1aae92c19d55, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e89a03c04fc54776a90dda002d71baea, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ea2476f9758a4c4db36e732e09c59777, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/96b92391fb2d49178545e517640ad2a6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/bf7d0c8f8164456e9155a40ae2a38417, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5f19b624b8c849f0bab3c7b41531861f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ddc3fb571cd846e692273e5fe4293141, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/cec5734b87c2451097029576dc7b539a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/00535eb7547642fea9d93992021e609d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/d0d665c60c6443c7a755f57d6c01a81e] to archive 2024-11-09T21:18:03,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:18:03,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f0e07846f13469e81874e96b6d776d1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f0e07846f13469e81874e96b6d776d1 2024-11-09T21:18:03,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9dad551c1dce4f8eb6990829c915ffe1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9dad551c1dce4f8eb6990829c915ffe1 2024-11-09T21:18:03,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b118933bb9f7452194451e7d2fa20e03 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b118933bb9f7452194451e7d2fa20e03 2024-11-09T21:18:03,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/98474b11e4374f92acdeae72f6cf93a8 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/98474b11e4374f92acdeae72f6cf93a8 2024-11-09T21:18:03,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/76b41ded0c1649d18a2ed94c92f6b74c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/76b41ded0c1649d18a2ed94c92f6b74c 2024-11-09T21:18:03,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8fda6af462a44411820007c99996f709 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8fda6af462a44411820007c99996f709 2024-11-09T21:18:03,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/100d6af7f3ff4b279996f4d10c4cb907 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/100d6af7f3ff4b279996f4d10c4cb907 2024-11-09T21:18:03,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9fe2f32525884117bb18e9c3695b58e0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/9fe2f32525884117bb18e9c3695b58e0 2024-11-09T21:18:03,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/216713a0def8487ab02cffec1afff9cc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/216713a0def8487ab02cffec1afff9cc 2024-11-09T21:18:03,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e38d25a6abdd4311bde8c14e60edb81d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e38d25a6abdd4311bde8c14e60edb81d 2024-11-09T21:18:03,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/345e4ae51fcd4455b3357b061f4ad525 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/345e4ae51fcd4455b3357b061f4ad525 2024-11-09T21:18:03,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5a244fb100ec4043a41f4332a66774ef to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5a244fb100ec4043a41f4332a66774ef 2024-11-09T21:18:03,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/6fc49537df0245bdbff719c66d2f5665 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/6fc49537df0245bdbff719c66d2f5665 2024-11-09T21:18:03,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f65add241ad427586c94dd21d0fe961 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/8f65add241ad427586c94dd21d0fe961 2024-11-09T21:18:03,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/278d8106963841878e835dfc0d20451a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/278d8106963841878e835dfc0d20451a 2024-11-09T21:18:03,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b8378f5b32f948b4b211b3f3ace502f7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/b8378f5b32f948b4b211b3f3ace502f7 2024-11-09T21:18:03,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ed7ee584797245afb43cc0824ae0917b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ed7ee584797245afb43cc0824ae0917b 2024-11-09T21:18:03,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/fcf544f02e9043f7af5f4b121055acf0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/fcf544f02e9043f7af5f4b121055acf0 2024-11-09T21:18:03,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/09ecf9aecaa54bc389b1a63d0461038e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/09ecf9aecaa54bc389b1a63d0461038e 2024-11-09T21:18:03,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a211c12c07994d94bb1d46507a0b943f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a211c12c07994d94bb1d46507a0b943f 2024-11-09T21:18:03,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/c137d9ee73ab423cbce1316ad2200f6e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/c137d9ee73ab423cbce1316ad2200f6e 2024-11-09T21:18:03,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/1ce8782518c048cfb7db14b55216b1d0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/1ce8782518c048cfb7db14b55216b1d0 2024-11-09T21:18:03,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/28ee5ba309cf4e01bc35b1944cc42fb3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/28ee5ba309cf4e01bc35b1944cc42fb3 2024-11-09T21:18:03,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ec28b77d23304b61af6e1aae92c19d55 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ec28b77d23304b61af6e1aae92c19d55 2024-11-09T21:18:03,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e89a03c04fc54776a90dda002d71baea to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/e89a03c04fc54776a90dda002d71baea 2024-11-09T21:18:03,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ea2476f9758a4c4db36e732e09c59777 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ea2476f9758a4c4db36e732e09c59777 2024-11-09T21:18:03,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/96b92391fb2d49178545e517640ad2a6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/96b92391fb2d49178545e517640ad2a6 2024-11-09T21:18:03,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/bf7d0c8f8164456e9155a40ae2a38417 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/bf7d0c8f8164456e9155a40ae2a38417 2024-11-09T21:18:03,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5f19b624b8c849f0bab3c7b41531861f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/5f19b624b8c849f0bab3c7b41531861f 2024-11-09T21:18:03,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ddc3fb571cd846e692273e5fe4293141 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/ddc3fb571cd846e692273e5fe4293141 2024-11-09T21:18:03,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/cec5734b87c2451097029576dc7b539a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/cec5734b87c2451097029576dc7b539a 2024-11-09T21:18:03,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/00535eb7547642fea9d93992021e609d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/00535eb7547642fea9d93992021e609d 2024-11-09T21:18:03,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/d0d665c60c6443c7a755f57d6c01a81e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/d0d665c60c6443c7a755f57d6c01a81e 2024-11-09T21:18:03,142 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/recovered.edits/472.seqid, newMaxSeqId=472, maxSeqId=1 2024-11-09T21:18:03,142 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. 2024-11-09T21:18:03,142 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] regionserver.HRegion(1676): Region close journal for 2b19020342a14d63e946c9297523dd88: Waiting for close lock at 1731187081829Running coprocessor pre-close hooks at 1731187081829Disabling compacts and flushes for region at 1731187081829Disabling writes for close at 1731187081829Obtaining lock to block concurrent updates at 1731187081829Preparing flush snapshotting stores in 2b19020342a14d63e946c9297523dd88 at 1731187081829Finished memstore snapshotting TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731187081829Flushing stores of TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88. at 1731187081830 (+1 ms)Flushing 2b19020342a14d63e946c9297523dd88/A: creating writer at 1731187081830Flushing 2b19020342a14d63e946c9297523dd88/A: appending metadata at 1731187081832 (+2 ms)Flushing 2b19020342a14d63e946c9297523dd88/A: closing flushed file at 1731187081832Flushing 2b19020342a14d63e946c9297523dd88/B: creating writer at 1731187082239 (+407 ms)Flushing 2b19020342a14d63e946c9297523dd88/B: appending metadata at 1731187082241 (+2 ms)Flushing 2b19020342a14d63e946c9297523dd88/B: closing flushed file at 1731187082241Flushing 2b19020342a14d63e946c9297523dd88/C: creating writer at 1731187082647 (+406 ms)Flushing 2b19020342a14d63e946c9297523dd88/C: appending metadata at 1731187082649 (+2 ms)Flushing 2b19020342a14d63e946c9297523dd88/C: closing flushed file at 1731187082649Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a044029: reopening flushed file at 1731187083055 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7222bd95: reopening flushed file at 1731187083058 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@475df320: reopening flushed file at 1731187083061 (+3 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 2b19020342a14d63e946c9297523dd88 in 1235ms, sequenceid=469, compaction requested=true at 1731187083064 (+3 ms)Writing region close event to WAL at 1731187083139 (+75 ms)Running coprocessor post-close hooks at 1731187083142 (+3 ms)Closed at 1731187083142 2024-11-09T21:18:03,144 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=152}] handler.UnassignRegionHandler(157): Closed 2b19020342a14d63e946c9297523dd88 2024-11-09T21:18:03,144 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=151 updating hbase:meta row=2b19020342a14d63e946c9297523dd88, regionState=CLOSED 2024-11-09T21:18:03,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:18:03,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-09T21:18:03,147 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; CloseRegionProcedure 2b19020342a14d63e946c9297523dd88, server=9514d6d5e5be,38413,1731186934857 in 1.4690 sec 2024-11-09T21:18:03,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-09T21:18:03,148 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2b19020342a14d63e946c9297523dd88, UNASSIGN in 1.4730 sec 2024-11-09T21:18:03,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-09T21:18:03,150 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.4750 sec 2024-11-09T21:18:03,151 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187083150"}]},"ts":"1731187083150"} 2024-11-09T21:18:03,152 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T21:18:03,152 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T21:18:03,153 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.4840 sec 2024-11-09T21:18:03,223 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:18:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-11-09T21:18:03,816 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:03,816 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T21:18:03,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,818 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=153, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-09T21:18:03,819 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=153, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,820 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88 2024-11-09T21:18:03,822 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/recovered.edits] 2024-11-09T21:18:03,825 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/2ef74888978f40c3b790f59a24a8838d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/2ef74888978f40c3b790f59a24a8838d 2024-11-09T21:18:03,826 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/5e795612f8b74462a90cd3ef88962234 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/5e795612f8b74462a90cd3ef88962234 2024-11-09T21:18:03,827 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d925034044154c9e9e4107c6311d4a93 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/A/d925034044154c9e9e4107c6311d4a93 2024-11-09T21:18:03,833 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1950959ca03841eebffd0844c81ab7cb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/1950959ca03841eebffd0844c81ab7cb 2024-11-09T21:18:03,834 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8947feb1db984c24bc34fa1d758b0196 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/8947feb1db984c24bc34fa1d758b0196 2024-11-09T21:18:03,835 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/970132cb394a452eb170996bd37f200b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/B/970132cb394a452eb170996bd37f200b 2024-11-09T21:18:03,837 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/361ff29e17214a278e0bb7b871dfde17 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/361ff29e17214a278e0bb7b871dfde17 2024-11-09T21:18:03,837 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/3ec9f48e6dba41af951937459d5c2600 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/3ec9f48e6dba41af951937459d5c2600 2024-11-09T21:18:03,838 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a202e6bbe5594275b00e44565a75f451 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/C/a202e6bbe5594275b00e44565a75f451 2024-11-09T21:18:03,840 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/recovered.edits/472.seqid to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88/recovered.edits/472.seqid 2024-11-09T21:18:03,840 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/2b19020342a14d63e946c9297523dd88 2024-11-09T21:18:03,840 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T21:18:03,842 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=153, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,844 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T21:18:03,845 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T21:18:03,846 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=153, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,846 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T21:18:03,847 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731187083846"}]},"ts":"9223372036854775807"} 2024-11-09T21:18:03,848 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T21:18:03,848 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2b19020342a14d63e946c9297523dd88, NAME => 'TestAcidGuarantees,,1731187057243.2b19020342a14d63e946c9297523dd88.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T21:18:03,848 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T21:18:03,849 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731187083848"}]},"ts":"9223372036854775807"} 2024-11-09T21:18:03,850 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T21:18:03,850 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=153, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 33 msec 2024-11-09T21:18:03,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-11-09T21:18:03,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T21:18:03,927 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:03,936 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=251 (was 251), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=494 (was 437) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3149 (was 3589) 2024-11-09T21:18:03,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=251, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=494, ProcessCount=11, AvailableMemoryMB=3149 2024-11-09T21:18:03,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:18:03,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:18:03,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:03,948 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T21:18:03,948 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:03,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 154 2024-11-09T21:18:03,949 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T21:18:03,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T21:18:03,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742375_1551 (size=1061) 2024-11-09T21:18:04,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T21:18:04,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T21:18:04,355 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982 2024-11-09T21:18:04,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742376_1552 (size=53) 2024-11-09T21:18:04,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T21:18:04,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T21:18:04,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:18:04,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 172f8346f592a41a81f031b40097d85b, disabling compactions & flushes 2024-11-09T21:18:04,759 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:04,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:04,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. after waiting 0 ms 2024-11-09T21:18:04,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:04,759 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:04,759 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 172f8346f592a41a81f031b40097d85b: Waiting for close lock at 1731187084759Disabling compacts and flushes for region at 1731187084759Disabling writes for close at 1731187084759Writing region close event to WAL at 1731187084759Closed at 1731187084759 2024-11-09T21:18:04,760 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T21:18:04,761 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731187084760"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731187084760"}]},"ts":"1731187084760"} 2024-11-09T21:18:04,762 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-09T21:18:04,762 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T21:18:04,762 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187084762"}]},"ts":"1731187084762"} 2024-11-09T21:18:04,763 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-09T21:18:04,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, ASSIGN}] 2024-11-09T21:18:04,764 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, ASSIGN 2024-11-09T21:18:04,765 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, ASSIGN; state=OFFLINE, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=false 2024-11-09T21:18:04,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:04,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=155, ppid=154, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, ASSIGN because future has completed 2024-11-09T21:18:04,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:18:05,072 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:05,072 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7752): Opening region: {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:18:05,072 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,072 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:18:05,072 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7794): checking encryption for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,072 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(7797): checking classloading for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,073 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,074 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:18:05,074 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 172f8346f592a41a81f031b40097d85b columnFamilyName A 2024-11-09T21:18:05,074 DEBUG [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:05,075 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(327): Store=172f8346f592a41a81f031b40097d85b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:18:05,075 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,076 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:18:05,076 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 172f8346f592a41a81f031b40097d85b columnFamilyName B 2024-11-09T21:18:05,076 DEBUG [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:05,076 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(327): Store=172f8346f592a41a81f031b40097d85b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:18:05,076 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,077 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:18:05,077 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 172f8346f592a41a81f031b40097d85b columnFamilyName C 2024-11-09T21:18:05,077 DEBUG [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:05,078 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(327): Store=172f8346f592a41a81f031b40097d85b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:18:05,078 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:05,078 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1038): replaying wal for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,078 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,079 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,079 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1048): stopping wal replay for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,079 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1060): Cleaning up temporary data for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,080 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:18:05,081 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1093): writing seq id for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,082 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T21:18:05,082 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1114): Opened 172f8346f592a41a81f031b40097d85b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61766753, jitterRate=-0.0796036571264267}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:18:05,082 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:05,083 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegion(1006): Region open journal for 172f8346f592a41a81f031b40097d85b: Running coprocessor pre-open hook at 1731187085072Writing region info on filesystem at 1731187085072Initializing all the Stores at 1731187085073 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187085073Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187085073Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187085073Cleaning up temporary data from old regions at 1731187085079 (+6 ms)Running coprocessor post-open hooks at 1731187085082 (+3 ms)Region opened successfully at 1731187085083 (+1 ms) 2024-11-09T21:18:05,084 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., pid=156, masterSystemTime=1731187085069 2024-11-09T21:18:05,085 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:05,085 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=156}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:05,086 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=155 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=OPEN, openSeqNum=2, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:05,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T21:18:05,087 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=156, ppid=155, state=RUNNABLE, hasLock=false; OpenRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:18:05,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-09T21:18:05,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; OpenRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 in 171 msec 2024-11-09T21:18:05,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-09T21:18:05,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, ASSIGN in 326 msec 2024-11-09T21:18:05,091 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T21:18:05,091 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187085091"}]},"ts":"1731187085091"} 2024-11-09T21:18:05,092 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-09T21:18:05,092 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=154, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T21:18:05,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1460 sec 2024-11-09T21:18:06,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-09T21:18:06,096 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:06,097 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4826bbfc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:06,097 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:06,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:06,098 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60551293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:06,099 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46322, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:06,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b6828c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:06,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:06,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:06,101 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:06,101 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:06,102 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T21:18:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9514d6d5e5be,34371,1731186934105 2024-11-09T21:18:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3d69801a 2024-11-09T21:18:06,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T21:18:06,103 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46324, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T21:18:06,105 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-09T21:18:06,105 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T21:18:06,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=157, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:06,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742377_1553 (size=1097) 2024-11-09T21:18:06,516 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-09T21:18:06,516 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-09T21:18:06,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:18:06,520 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, REOPEN/MOVE}] 2024-11-09T21:18:06,520 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, REOPEN/MOVE 2024-11-09T21:18:06,521 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:06,522 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, REOPEN/MOVE because future has completed 2024-11-09T21:18:06,522 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:18:06,522 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; CloseRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:18:06,674 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(122): Close 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,674 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:18:06,674 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1722): Closing 172f8346f592a41a81f031b40097d85b, disabling compactions & flushes 2024-11-09T21:18:06,674 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,674 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,674 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. after waiting 0 ms 2024-11-09T21:18:06,674 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,677 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T21:18:06,677 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,677 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegion(1676): Region close journal for 172f8346f592a41a81f031b40097d85b: Waiting for close lock at 1731187086674Running coprocessor pre-close hooks at 1731187086674Disabling compacts and flushes for region at 1731187086674Disabling writes for close at 1731187086674Writing region close event to WAL at 1731187086674Running coprocessor post-close hooks at 1731187086677 (+3 ms)Closed at 1731187086677 2024-11-09T21:18:06,677 WARN [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] regionserver.HRegionServer(3299): Not adding moved region record: 172f8346f592a41a81f031b40097d85b to self. 2024-11-09T21:18:06,679 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=160}] handler.UnassignRegionHandler(157): Closed 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,679 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=CLOSED 2024-11-09T21:18:06,680 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=160, ppid=159, state=RUNNABLE, hasLock=false; CloseRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:18:06,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-11-09T21:18:06,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; CloseRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 in 159 msec 2024-11-09T21:18:06,683 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, REOPEN/MOVE; state=CLOSED, location=9514d6d5e5be,38413,1731186934857; forceNewPlan=false, retain=true 2024-11-09T21:18:06,833 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=OPENING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:06,835 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=158, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, REOPEN/MOVE because future has completed 2024-11-09T21:18:06,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:18:06,988 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,988 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7752): Opening region: {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} 2024-11-09T21:18:06,989 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,989 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T21:18:06,989 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7794): checking encryption for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,989 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(7797): checking classloading for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,990 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,990 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:18:06,990 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 172f8346f592a41a81f031b40097d85b columnFamilyName A 2024-11-09T21:18:06,991 DEBUG [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:06,991 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(327): Store=172f8346f592a41a81f031b40097d85b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:18:06,992 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,992 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:18:06,992 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 172f8346f592a41a81f031b40097d85b columnFamilyName B 2024-11-09T21:18:06,992 DEBUG [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:06,992 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(327): Store=172f8346f592a41a81f031b40097d85b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:18:06,992 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,993 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-09T21:18:06,993 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 172f8346f592a41a81f031b40097d85b columnFamilyName C 2024-11-09T21:18:06,993 DEBUG [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:06,993 INFO [StoreOpener-172f8346f592a41a81f031b40097d85b-1 {}] regionserver.HStore(327): Store=172f8346f592a41a81f031b40097d85b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T21:18:06,993 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,993 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1038): replaying wal for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,994 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,994 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,995 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1048): stopping wal replay for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,995 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1060): Cleaning up temporary data for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,995 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-09T21:18:06,996 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1093): writing seq id for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,997 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1114): Opened 172f8346f592a41a81f031b40097d85b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66054500, jitterRate=-0.01571124792098999}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-09T21:18:06,997 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:06,997 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegion(1006): Region open journal for 172f8346f592a41a81f031b40097d85b: Running coprocessor pre-open hook at 1731187086989Writing region info on filesystem at 1731187086989Initializing all the Stores at 1731187086989Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187086989Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187086990 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731187086990Cleaning up temporary data from old regions at 1731187086995 (+5 ms)Running coprocessor post-open hooks at 1731187086997 (+2 ms)Region opened successfully at 1731187086997 2024-11-09T21:18:06,998 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., pid=161, masterSystemTime=1731187086986 2024-11-09T21:18:06,999 DEBUG [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:06,999 INFO [RS_OPEN_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_OPEN_REGION, pid=161}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,000 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=159 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=OPEN, openSeqNum=5, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=161, ppid=159, state=RUNNABLE, hasLock=false; OpenRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:18:07,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=159 2024-11-09T21:18:07,003 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=159, state=SUCCESS, hasLock=false; OpenRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 in 166 msec 2024-11-09T21:18:07,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-09T21:18:07,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, REOPEN/MOVE in 483 msec 2024-11-09T21:18:07,006 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-09T21:18:07,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 487 msec 2024-11-09T21:18:07,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 902 msec 2024-11-09T21:18:07,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=157 2024-11-09T21:18:07,009 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:07,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@604e524e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,009 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,010 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5012ad90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,011 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46344, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2eb7744e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f36d46f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,012 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a758a18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,013 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46358, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,013 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3381441e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,014 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e61df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,014 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,014 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,014 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,014 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fcf638f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,015 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,015 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@203ae9e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,016 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885acdb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,016 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,017 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@651aec62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,017 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,017 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46408, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,018 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@431a89d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,018 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6cc99027, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,018 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,018 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,019 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c23774, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,020 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a0d61db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,021 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3420b4f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,021 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,021 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,021 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@56714cf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,021 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,022 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46442, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,022 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59ee387e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,023 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d1b2f09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,023 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,023 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17870e81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,024 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@603dc9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,025 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1102b71d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,025 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,026 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,026 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2757fb1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,027 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,027 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46500, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,028 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d6b0f8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,028 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22ff4806, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,028 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,028 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,029 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b55ba9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,029 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,030 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46518, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24875312, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,031 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a8677bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,031 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9514d6d5e5be,34371,-1 for getting cluster id 2024-11-09T21:18:07,031 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T21:18:07,031 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '131b0082-dcda-4525-9e4a-2e80d2847bb6' 2024-11-09T21:18:07,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T21:18:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "131b0082-dcda-4525-9e4a-2e80d2847bb6" 2024-11-09T21:18:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74a81b18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9514d6d5e5be,34371,-1] 2024-11-09T21:18:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T21:18:07,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:07,032 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46524, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T21:18:07,033 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4d9765, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T21:18:07,033 DEBUG [Thread-2466 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,034 DEBUG [Thread-2475 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,034 DEBUG [Thread-2460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,034 DEBUG [Thread-2471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,034 DEBUG [Thread-2473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,034 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,035 DEBUG [Thread-2473 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,035 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,035 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33134, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33162, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,036 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,036 DEBUG [Thread-2458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,037 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,038 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,039 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:07,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-09T21:18:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:07,045 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:07,048 DEBUG [Thread-2464 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,048 DEBUG [Thread-2477 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,049 DEBUG [Thread-2462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,049 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,050 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,050 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33194, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,052 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,056 DEBUG [Thread-2469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T21:18:07,057 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9514d6d5e5be,38413,1731186934857, seqNum=-1] 2024-11-09T21:18:07,058 DEBUG [Thread-2469 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T21:18:07,059 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33204, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T21:18:07,061 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5] 2024-11-09T21:18:07,064 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:18:07,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-09T21:18:07,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:07,066 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:18:07,067 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:18:07,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:18:07,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731187087062, max timestamp=1731187087075Num uniques 859; 2024-11-09T21:18:07,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731187087062, max timestamp=1731187087075Num uniques 859; 2024-11-09T21:18:07,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731187087062, max timestamp=1731187087075Num uniques 859; 2024-11-09T21:18:07,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:18:07,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:18:07,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-09T21:18:07,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109689e1a5e6fb44f02aa9e5b66b5c66c15_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187087043/Put/seqid=0 2024-11-09T21:18:07,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742378_1554 (size=12154) 2024-11-09T21:18:07,102 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187147095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187147095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187147096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187147098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,105 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109689e1a5e6fb44f02aa9e5b66b5c66c15_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109689e1a5e6fb44f02aa9e5b66b5c66c15_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:07,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,106 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/418dd56e4a5046d2b2836632e0958e96, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187147101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,106 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/418dd56e4a5046d2b2836632e0958e96 is 175, key is test_row_0/A:col10/1731187087043/Put/seqid=0 2024-11-09T21:18:07,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742379_1555 (size=30955) 2024-11-09T21:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187147207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187147207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187147207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187147207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,209 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187147216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,372 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187147416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187147416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,417 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187147416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187147416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,418 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187147426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,429 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,522 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/418dd56e4a5046d2b2836632e0958e96 2024-11-09T21:18:07,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,544 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/fd7b6bbb2cfa461ba8693fd0a6c89e84 is 50, key is test_row_0/B:col10/1731187087043/Put/seqid=0 2024-11-09T21:18:07,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742380_1556 (size=12001) 2024-11-09T21:18:07,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/fd7b6bbb2cfa461ba8693fd0a6c89e84 2024-11-09T21:18:07,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/8b115dd66441482582791ac33e967ba2 is 50, key is test_row_0/C:col10/1731187087043/Put/seqid=0 2024-11-09T21:18:07,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742381_1557 (size=12001) 2024-11-09T21:18:07,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:07,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187147727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187147727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187147727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187147727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187147736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:07,740 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:07,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:07,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/8b115dd66441482582791ac33e967ba2 2024-11-09T21:18:07,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/418dd56e4a5046d2b2836632e0958e96 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96 2024-11-09T21:18:07,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96, entries=150, sequenceid=14, filesize=30.2 K 2024-11-09T21:18:07,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/fd7b6bbb2cfa461ba8693fd0a6c89e84 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/fd7b6bbb2cfa461ba8693fd0a6c89e84 2024-11-09T21:18:07,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:07,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/fd7b6bbb2cfa461ba8693fd0a6c89e84, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T21:18:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:07,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:07,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/8b115dd66441482582791ac33e967ba2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8b115dd66441482582791ac33e967ba2 2024-11-09T21:18:07,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:07,988 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8b115dd66441482582791ac33e967ba2, entries=150, sequenceid=14, filesize=11.7 K 2024-11-09T21:18:07,989 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 172f8346f592a41a81f031b40097d85b in 944ms, sequenceid=14, compaction requested=false 2024-11-09T21:18:07,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:08,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:08,137 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:08,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b3a095165af0460b82e0c5272f495c59_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187087100/Put/seqid=0 2024-11-09T21:18:08,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742382_1558 (size=12154) 2024-11-09T21:18:08,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:08,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:08,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187148244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187148245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187148246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187148246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187148246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,251 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187148357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187148357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187148357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187148357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187148567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187148567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187148567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187148567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,568 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:08,574 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b3a095165af0460b82e0c5272f495c59_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b3a095165af0460b82e0c5272f495c59_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:08,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/583d7f6eb615429180228ca8ce643401, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:08,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/583d7f6eb615429180228ca8ce643401 is 175, key is test_row_0/A:col10/1731187087100/Put/seqid=0 2024-11-09T21:18:08,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742383_1559 (size=30955) 2024-11-09T21:18:08,664 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T21:18:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,880 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187148877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187148877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187148877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187148877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,881 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,882 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:08,978 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/583d7f6eb615429180228ca8ce643401 2024-11-09T21:18:08,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/59561a936c1242d5ad036b4ad4c479c0 is 50, key is test_row_0/B:col10/1731187087100/Put/seqid=0 2024-11-09T21:18:08,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742384_1560 (size=12001) 2024-11-09T21:18:08,987 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/59561a936c1242d5ad036b4ad4c479c0 2024-11-09T21:18:08,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/2f3f517cb6eb41f688998300dd115fa2 is 50, key is test_row_0/C:col10/1731187087100/Put/seqid=0 2024-11-09T21:18:08,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742385_1561 (size=12001) 2024-11-09T21:18:09,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:09,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187149256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:09,258 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,259 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:09,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187149386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,389 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:09,398 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/2f3f517cb6eb41f688998300dd115fa2 2024-11-09T21:18:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187149396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187149396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:09,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:09,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187149396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:09,401 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:09,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/583d7f6eb615429180228ca8ce643401 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401 2024-11-09T21:18:09,405 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401, entries=150, sequenceid=49, filesize=30.2 K 2024-11-09T21:18:09,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/59561a936c1242d5ad036b4ad4c479c0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/59561a936c1242d5ad036b4ad4c479c0 2024-11-09T21:18:09,409 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/59561a936c1242d5ad036b4ad4c479c0, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T21:18:09,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/2f3f517cb6eb41f688998300dd115fa2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/2f3f517cb6eb41f688998300dd115fa2 2024-11-09T21:18:09,414 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/2f3f517cb6eb41f688998300dd115fa2, entries=150, sequenceid=49, filesize=11.7 K 2024-11-09T21:18:09,414 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 172f8346f592a41a81f031b40097d85b in 1277ms, sequenceid=49, compaction requested=false 2024-11-09T21:18:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:09,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-09T21:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-09T21:18:09,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-09T21:18:09,418 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3490 sec 2024-11-09T21:18:09,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.3540 sec 2024-11-09T21:18:10,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:10,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:18:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:10,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:10,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:10,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c76521819db348c3996bbdb332ced7c8_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:10,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742386_1562 (size=14594) 2024-11-09T21:18:10,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187150428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,434 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187150431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187150431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187150433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,438 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187150536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187150546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187150546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187150546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,548 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187150747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187150756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,758 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187150756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:10,759 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187150756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:10,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:10,817 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:10,820 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c76521819db348c3996bbdb332ced7c8_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c76521819db348c3996bbdb332ced7c8_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:10,820 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/9136ee24198a46ec8155683f7b56298b, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:10,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/9136ee24198a46ec8155683f7b56298b is 175, key is test_row_0/A:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:10,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742387_1563 (size=39549) 2024-11-09T21:18:11,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187151056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187151067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187151067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,069 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187151067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,070 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,224 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=61, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/9136ee24198a46ec8155683f7b56298b 2024-11-09T21:18:11,229 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/2a627725546441389897140d27f8673f is 50, key is test_row_0/B:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:11,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742388_1564 (size=12001) 2024-11-09T21:18:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-09T21:18:11,246 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:11,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:18:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-09T21:18:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T21:18:11,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:18:11,249 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:18:11,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:18:11,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187151276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,278 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:11,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T21:18:11,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T21:18:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:11,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:11,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:11,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:11,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T21:18:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T21:18:11,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187151567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,569 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187151576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187151576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187151576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:11,580 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:11,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/2a627725546441389897140d27f8673f 2024-11-09T21:18:11,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ba5059e8e3dc457799bcf9f5954fb056 is 50, key is test_row_0/C:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:11,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742389_1565 (size=12001) 2024-11-09T21:18:11,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ba5059e8e3dc457799bcf9f5954fb056 2024-11-09T21:18:11,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/9136ee24198a46ec8155683f7b56298b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b 2024-11-09T21:18:11,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b, entries=200, sequenceid=61, filesize=38.6 K 2024-11-09T21:18:11,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/2a627725546441389897140d27f8673f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2a627725546441389897140d27f8673f 2024-11-09T21:18:11,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2a627725546441389897140d27f8673f, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:18:11,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ba5059e8e3dc457799bcf9f5954fb056 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ba5059e8e3dc457799bcf9f5954fb056 2024-11-09T21:18:11,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ba5059e8e3dc457799bcf9f5954fb056, entries=150, sequenceid=61, filesize=11.7 K 2024-11-09T21:18:11,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 172f8346f592a41a81f031b40097d85b in 1253ms, sequenceid=61, compaction requested=true 2024-11-09T21:18:11,652 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:11,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:18:11,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:11,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:18:11,652 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:11,652 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:11,652 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:11,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:18:11,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:11,654 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:11,654 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:11,654 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/A is initiating minor compaction (all files) 2024-11-09T21:18:11,654 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/A in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,654 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=99.1 K 2024-11-09T21:18:11,654 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,654 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b] 2024-11-09T21:18:11,654 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/B is initiating minor compaction (all files) 2024-11-09T21:18:11,654 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/B in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,654 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/fd7b6bbb2cfa461ba8693fd0a6c89e84, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/59561a936c1242d5ad036b4ad4c479c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2a627725546441389897140d27f8673f] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=35.2 K 2024-11-09T21:18:11,655 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fd7b6bbb2cfa461ba8693fd0a6c89e84, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731187087041 2024-11-09T21:18:11,655 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 418dd56e4a5046d2b2836632e0958e96, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731187087041 2024-11-09T21:18:11,656 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 59561a936c1242d5ad036b4ad4c479c0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731187087085 2024-11-09T21:18:11,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 583d7f6eb615429180228ca8ce643401, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731187087085 2024-11-09T21:18:11,656 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a627725546441389897140d27f8673f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731187088237 2024-11-09T21:18:11,656 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9136ee24198a46ec8155683f7b56298b, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731187088237 2024-11-09T21:18:11,660 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:11,662 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#B#compaction#487 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:11,663 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/00d5457b990545558323668da39f9035 is 50, key is test_row_0/B:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:11,664 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024110976f8c721302c484f84c10a6902dc01de_172f8346f592a41a81f031b40097d85b store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:11,666 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024110976f8c721302c484f84c10a6902dc01de_172f8346f592a41a81f031b40097d85b, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:11,666 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024110976f8c721302c484f84c10a6902dc01de_172f8346f592a41a81f031b40097d85b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:11,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742390_1566 (size=12104) 2024-11-09T21:18:11,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742391_1567 (size=4469) 2024-11-09T21:18:11,707 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-09T21:18:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:11,707 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-09T21:18:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:11,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:11,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:11,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:11,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e4049cdbf7204218a0bcd422c2ff78a8_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187090429/Put/seqid=0 2024-11-09T21:18:11,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742392_1568 (size=12154) 2024-11-09T21:18:11,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T21:18:12,072 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/00d5457b990545558323668da39f9035 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/00d5457b990545558323668da39f9035 2024-11-09T21:18:12,075 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/B of 172f8346f592a41a81f031b40097d85b into 00d5457b990545558323668da39f9035(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:12,075 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:12,075 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/B, priority=13, startTime=1731187091652; duration=0sec 2024-11-09T21:18:12,075 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:12,075 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:B 2024-11-09T21:18:12,075 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:12,076 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:12,076 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/C is initiating minor compaction (all files) 2024-11-09T21:18:12,076 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/C in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:12,076 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8b115dd66441482582791ac33e967ba2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/2f3f517cb6eb41f688998300dd115fa2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ba5059e8e3dc457799bcf9f5954fb056] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=35.2 K 2024-11-09T21:18:12,077 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b115dd66441482582791ac33e967ba2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731187087041 2024-11-09T21:18:12,077 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3f517cb6eb41f688998300dd115fa2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731187087085 2024-11-09T21:18:12,077 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ba5059e8e3dc457799bcf9f5954fb056, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731187088237 2024-11-09T21:18:12,084 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#A#compaction#486 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:12,085 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/a606fd6909944da6ab6dcf683372a93e is 175, key is test_row_0/A:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:12,094 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#C#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:12,094 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/a6c4eaffa86748758d6ab23b4f1976ee is 50, key is test_row_0/C:col10/1731187088237/Put/seqid=0 2024-11-09T21:18:12,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742393_1569 (size=31058) 2024-11-09T21:18:12,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742394_1570 (size=12104) 2024-11-09T21:18:12,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:12,128 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109e4049cdbf7204218a0bcd422c2ff78a8_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e4049cdbf7204218a0bcd422c2ff78a8_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:12,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6765f5a650b8472eac4c5362ef8224dc, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:12,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6765f5a650b8472eac4c5362ef8224dc is 175, key is test_row_0/A:col10/1731187090429/Put/seqid=0 2024-11-09T21:18:12,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742395_1571 (size=30955) 2024-11-09T21:18:12,134 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6765f5a650b8472eac4c5362ef8224dc 2024-11-09T21:18:12,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/13177f786ee6435f9771579cef802950 is 50, key is test_row_0/B:col10/1731187090429/Put/seqid=0 2024-11-09T21:18:12,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742396_1572 (size=12001) 2024-11-09T21:18:12,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T21:18:12,506 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/a606fd6909944da6ab6dcf683372a93e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/a606fd6909944da6ab6dcf683372a93e 2024-11-09T21:18:12,509 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/A of 172f8346f592a41a81f031b40097d85b into a606fd6909944da6ab6dcf683372a93e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:12,509 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:12,509 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/A, priority=13, startTime=1731187091652; duration=0sec 2024-11-09T21:18:12,509 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:12,509 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:A 2024-11-09T21:18:12,517 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/a6c4eaffa86748758d6ab23b4f1976ee as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/a6c4eaffa86748758d6ab23b4f1976ee 2024-11-09T21:18:12,521 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/C of 172f8346f592a41a81f031b40097d85b into a6c4eaffa86748758d6ab23b4f1976ee(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:12,521 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:12,521 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/C, priority=13, startTime=1731187091652; duration=0sec 2024-11-09T21:18:12,521 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:12,521 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:C 2024-11-09T21:18:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:12,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:12,594 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/13177f786ee6435f9771579cef802950 2024-11-09T21:18:12,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187152594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,599 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/f2eb563dfdca46fcb25aaa923067e35d is 50, key is test_row_0/C:col10/1731187090429/Put/seqid=0 2024-11-09T21:18:12,600 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187152595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187152597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,601 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,602 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187152597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,602 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742397_1573 (size=12001) 2024-11-09T21:18:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187152706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187152706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,708 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187152706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,709 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187152706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,709 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187152916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187152916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,919 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187152917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:12,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187152917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:12,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/f2eb563dfdca46fcb25aaa923067e35d 2024-11-09T21:18:13,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6765f5a650b8472eac4c5362ef8224dc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc 2024-11-09T21:18:13,010 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc, entries=150, sequenceid=81, filesize=30.2 K 2024-11-09T21:18:13,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/13177f786ee6435f9771579cef802950 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/13177f786ee6435f9771579cef802950 2024-11-09T21:18:13,014 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/13177f786ee6435f9771579cef802950, entries=150, sequenceid=81, filesize=11.7 K 2024-11-09T21:18:13,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/f2eb563dfdca46fcb25aaa923067e35d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f2eb563dfdca46fcb25aaa923067e35d 2024-11-09T21:18:13,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f2eb563dfdca46fcb25aaa923067e35d, entries=150, sequenceid=81, filesize=11.7 K 2024-11-09T21:18:13,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 172f8346f592a41a81f031b40097d85b in 1311ms, sequenceid=81, compaction requested=false 2024-11-09T21:18:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-09T21:18:13,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-09T21:18:13,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-09T21:18:13,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7710 sec 2024-11-09T21:18:13,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.7750 sec 2024-11-09T21:18:13,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:13,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:18:13,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:13,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:13,228 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:13,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:13,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:13,229 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:13,234 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091b9bed6702ce48959a4e0091bf950e40_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:13,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742398_1574 (size=14594) 2024-11-09T21:18:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187153247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187153248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187153248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,252 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187153252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,257 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187153356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,357 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187153357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,361 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187153357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,362 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187153366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-09T21:18:13,396 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:13,397 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:18:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-09T21:18:13,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T21:18:13,401 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:18:13,402 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:18:13,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:18:13,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T21:18:13,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,554 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187153567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,571 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187153567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,572 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187153567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,572 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187153576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,641 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:13,645 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411091b9bed6702ce48959a4e0091bf950e40_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091b9bed6702ce48959a4e0091bf950e40_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:13,646 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e2e40e4d4a034a8eba439ff252506dfc, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:13,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e2e40e4d4a034a8eba439ff252506dfc is 175, key is test_row_0/A:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:13,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742399_1575 (size=39549) 2024-11-09T21:18:13,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T21:18:13,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:13,859 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:13,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187153877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187153877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187153877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,879 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:13,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:13,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187153886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:13,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:13,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:14,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T21:18:14,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=97, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e2e40e4d4a034a8eba439ff252506dfc 2024-11-09T21:18:14,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/5b81833f6365449ba021804fb093a963 is 50, key is test_row_0/B:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:14,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742400_1576 (size=12001) 2024-11-09T21:18:14,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:14,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187154386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:14,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187154386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,388 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:14,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:14,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187154387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:14,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:14,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187154396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:14,399 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:14,453 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-09T21:18:14,453 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-09T21:18:14,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,473 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/5b81833f6365449ba021804fb093a963 2024-11-09T21:18:14,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/10e0e19889164b828a320918ed98f12b is 50, key is test_row_0/C:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:14,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742401_1577 (size=12001) 2024-11-09T21:18:14,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T21:18:14,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:14,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:14,881 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/10e0e19889164b828a320918ed98f12b 2024-11-09T21:18:14,884 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e2e40e4d4a034a8eba439ff252506dfc as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc 2024-11-09T21:18:14,887 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc, entries=200, sequenceid=97, filesize=38.6 K 2024-11-09T21:18:14,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/5b81833f6365449ba021804fb093a963 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/5b81833f6365449ba021804fb093a963 2024-11-09T21:18:14,890 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/5b81833f6365449ba021804fb093a963, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:18:14,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/10e0e19889164b828a320918ed98f12b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/10e0e19889164b828a320918ed98f12b 2024-11-09T21:18:14,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/10e0e19889164b828a320918ed98f12b, entries=150, sequenceid=97, filesize=11.7 K 2024-11-09T21:18:14,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 172f8346f592a41a81f031b40097d85b in 1665ms, sequenceid=97, compaction requested=true 2024-11-09T21:18:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:18:14,893 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:14,893 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:18:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:14,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:18:14,894 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:14,894 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:14,894 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:14,894 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/A is initiating minor compaction (all files) 2024-11-09T21:18:14,894 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/B is initiating minor compaction (all files) 2024-11-09T21:18:14,894 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/B in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,894 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/A in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,894 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/00d5457b990545558323668da39f9035, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/13177f786ee6435f9771579cef802950, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/5b81833f6365449ba021804fb093a963] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=35.3 K 2024-11-09T21:18:14,894 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/a606fd6909944da6ab6dcf683372a93e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=99.2 K 2024-11-09T21:18:14,894 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,894 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/a606fd6909944da6ab6dcf683372a93e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc] 2024-11-09T21:18:14,894 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d5457b990545558323668da39f9035, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731187088237 2024-11-09T21:18:14,895 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 13177f786ee6435f9771579cef802950, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731187090429 2024-11-09T21:18:14,895 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting a606fd6909944da6ab6dcf683372a93e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731187088237 2024-11-09T21:18:14,895 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b81833f6365449ba021804fb093a963, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731187092593 2024-11-09T21:18:14,895 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6765f5a650b8472eac4c5362ef8224dc, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731187090429 2024-11-09T21:18:14,895 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2e40e4d4a034a8eba439ff252506dfc, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731187092593 2024-11-09T21:18:14,905 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:14,906 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#B#compaction#495 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:14,906 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/6f475706008948789ea1c931d51fd83f is 50, key is test_row_0/B:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:14,907 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109af997295868141e3b95333b4ef86c5f9_172f8346f592a41a81f031b40097d85b store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:14,909 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109af997295868141e3b95333b4ef86c5f9_172f8346f592a41a81f031b40097d85b, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:14,909 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109af997295868141e3b95333b4ef86c5f9_172f8346f592a41a81f031b40097d85b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:14,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742402_1578 (size=12207) 2024-11-09T21:18:14,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-09T21:18:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:14,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T21:18:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:14,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:14,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:14,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742403_1579 (size=4469) 2024-11-09T21:18:14,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095e9b23449b244d2c8e21cd502fe9c02e_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187093235/Put/seqid=0 2024-11-09T21:18:14,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742404_1580 (size=12154) 2024-11-09T21:18:15,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:15,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:15,325 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/6f475706008948789ea1c931d51fd83f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/6f475706008948789ea1c931d51fd83f 2024-11-09T21:18:15,329 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/B of 172f8346f592a41a81f031b40097d85b into 6f475706008948789ea1c931d51fd83f(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:15,329 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:15,329 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/B, priority=13, startTime=1731187094893; duration=0sec 2024-11-09T21:18:15,329 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:15,329 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:B 2024-11-09T21:18:15,329 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:15,330 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:15,330 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/C is initiating minor compaction (all files) 2024-11-09T21:18:15,330 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/C in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:15,330 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/a6c4eaffa86748758d6ab23b4f1976ee, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f2eb563dfdca46fcb25aaa923067e35d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/10e0e19889164b828a320918ed98f12b] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=35.3 K 2024-11-09T21:18:15,330 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting a6c4eaffa86748758d6ab23b4f1976ee, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731187088237 2024-11-09T21:18:15,330 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f2eb563dfdca46fcb25aaa923067e35d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731187090429 2024-11-09T21:18:15,330 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 10e0e19889164b828a320918ed98f12b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731187092593 2024-11-09T21:18:15,336 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#C#compaction#498 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:15,336 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/1611186e516040f99aba56113ebf92b4 is 50, key is test_row_0/C:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:15,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742405_1581 (size=12207) 2024-11-09T21:18:15,344 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/1611186e516040f99aba56113ebf92b4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/1611186e516040f99aba56113ebf92b4 2024-11-09T21:18:15,347 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#A#compaction#496 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:15,347 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/d84ec7db34754ccab2195a5936f3cbfa is 175, key is test_row_0/A:col10/1731187093227/Put/seqid=0 2024-11-09T21:18:15,347 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/C of 172f8346f592a41a81f031b40097d85b into 1611186e516040f99aba56113ebf92b4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:15,347 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:15,347 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/C, priority=13, startTime=1731187094893; duration=0sec 2024-11-09T21:18:15,348 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:15,348 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:C 2024-11-09T21:18:15,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187155346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,351 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742406_1582 (size=31161) 2024-11-09T21:18:15,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:15,373 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411095e9b23449b244d2c8e21cd502fe9c02e_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095e9b23449b244d2c8e21cd502fe9c02e_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:15,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/dfba6c4e565949aabe1327e7db1f8d35, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:15,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/dfba6c4e565949aabe1327e7db1f8d35 is 175, key is test_row_0/A:col10/1731187093235/Put/seqid=0 2024-11-09T21:18:15,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742407_1583 (size=30955) 2024-11-09T21:18:15,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=116, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/dfba6c4e565949aabe1327e7db1f8d35 2024-11-09T21:18:15,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187155397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187155397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,399 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,399 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,399 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_0' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,400 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/15b774a6c5734b98a77e9c19cfac0b37 is 50, key is test_row_0/B:col10/1731187093235/Put/seqid=0 2024-11-09T21:18:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187155406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187155407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,407 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,407 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742408_1584 (size=12001) 2024-11-09T21:18:15,430 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/15b774a6c5734b98a77e9c19cfac0b37 2024-11-09T21:18:15,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/af787d025fbd4a049cef3f4baa7febdb is 50, key is test_row_0/C:col10/1731187093235/Put/seqid=0 2024-11-09T21:18:15,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742409_1585 (size=12001) 2024-11-09T21:18:15,450 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/af787d025fbd4a049cef3f4baa7febdb 2024-11-09T21:18:15,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/dfba6c4e565949aabe1327e7db1f8d35 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35 2024-11-09T21:18:15,455 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35, entries=150, sequenceid=116, filesize=30.2 K 2024-11-09T21:18:15,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/15b774a6c5734b98a77e9c19cfac0b37 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/15b774a6c5734b98a77e9c19cfac0b37 2024-11-09T21:18:15,458 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/15b774a6c5734b98a77e9c19cfac0b37, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T21:18:15,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/af787d025fbd4a049cef3f4baa7febdb as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/af787d025fbd4a049cef3f4baa7febdb 2024-11-09T21:18:15,460 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187155457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,460 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,462 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/af787d025fbd4a049cef3f4baa7febdb, entries=150, sequenceid=116, filesize=11.7 K 2024-11-09T21:18:15,463 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 172f8346f592a41a81f031b40097d85b in 532ms, sequenceid=116, compaction requested=false 2024-11-09T21:18:15,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:15,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:15,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-09T21:18:15,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-09T21:18:15,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-09T21:18:15,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0620 sec 2024-11-09T21:18:15,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 2.0690 sec 2024-11-09T21:18:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-09T21:18:15,547 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:15,547 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:18:15,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-09T21:18:15,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:15,549 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:18:15,549 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:18:15,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:18:15,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:15,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-09T21:18:15,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:15,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:15,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:15,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:15,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:15,669 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:15,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a725183e2b9e4aa09ec38ea700bc830c_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:15,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742410_1586 (size=14694) 2024-11-09T21:18:15,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:15,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:15,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:15,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:15,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:15,754 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187155749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,757 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/d84ec7db34754ccab2195a5936f3cbfa as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d84ec7db34754ccab2195a5936f3cbfa 2024-11-09T21:18:15,760 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/A of 172f8346f592a41a81f031b40097d85b into d84ec7db34754ccab2195a5936f3cbfa(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:15,760 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:15,760 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/A, priority=13, startTime=1731187094893; duration=0sec 2024-11-09T21:18:15,760 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:15,760 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:A 2024-11-09T21:18:15,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:15,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:15,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:15,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:15,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:15,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:15,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:15,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:15,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187155856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:15,859 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:15,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:16,007 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,007 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187156066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:16,069 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:16,079 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:16,082 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a725183e2b9e4aa09ec38ea700bc830c_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a725183e2b9e4aa09ec38ea700bc830c_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:16,083 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/04f7cdbf23af4015875c63205f9e1ad0, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:16,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/04f7cdbf23af4015875c63205f9e1ad0 is 175, key is test_row_0/A:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:16,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742411_1587 (size=39649) 2024-11-09T21:18:16,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:16,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187156376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:16,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:16,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,502 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/04f7cdbf23af4015875c63205f9e1ad0 2024-11-09T21:18:16,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/38d41d9a457e43289cb7d199dd389d80 is 50, key is test_row_0/B:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:16,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742412_1588 (size=12051) 2024-11-09T21:18:16,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:16,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,887 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:16,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187156886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:16,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:16,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/38d41d9a457e43289cb7d199dd389d80 2024-11-09T21:18:16,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/30ce283e1c2547ac8e814c4156cbbe23 is 50, key is test_row_0/C:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:16,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742413_1589 (size=12051) 2024-11-09T21:18:16,953 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:16,953 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:16,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:17,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:17,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:17,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:17,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,258 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,319 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/30ce283e1c2547ac8e814c4156cbbe23 2024-11-09T21:18:17,322 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/04f7cdbf23af4015875c63205f9e1ad0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0 2024-11-09T21:18:17,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0, entries=200, sequenceid=132, filesize=38.7 K 2024-11-09T21:18:17,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/38d41d9a457e43289cb7d199dd389d80 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/38d41d9a457e43289cb7d199dd389d80 2024-11-09T21:18:17,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/38d41d9a457e43289cb7d199dd389d80, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:18:17,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/30ce283e1c2547ac8e814c4156cbbe23 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/30ce283e1c2547ac8e814c4156cbbe23 2024-11-09T21:18:17,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/30ce283e1c2547ac8e814c4156cbbe23, entries=150, sequenceid=132, filesize=11.8 K 2024-11-09T21:18:17,341 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 172f8346f592a41a81f031b40097d85b in 1672ms, sequenceid=132, compaction requested=true 2024-11-09T21:18:17,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:17,342 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:17,343 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:18:17,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:17,344 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:17,344 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:18:17,344 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/A is initiating minor compaction (all files) 2024-11-09T21:18:17,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:17,345 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/A in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,345 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d84ec7db34754ccab2195a5936f3cbfa, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=99.4 K 2024-11-09T21:18:17,345 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d84ec7db34754ccab2195a5936f3cbfa, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0] 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/B is initiating minor compaction (all files) 2024-11-09T21:18:17,345 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/B in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,345 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/6f475706008948789ea1c931d51fd83f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/15b774a6c5734b98a77e9c19cfac0b37, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/38d41d9a457e43289cb7d199dd389d80] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=35.4 K 2024-11-09T21:18:17,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting d84ec7db34754ccab2195a5936f3cbfa, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731187092593 2024-11-09T21:18:17,345 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f475706008948789ea1c931d51fd83f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731187092593 2024-11-09T21:18:17,345 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfba6c4e565949aabe1327e7db1f8d35, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731187093235 2024-11-09T21:18:17,346 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 15b774a6c5734b98a77e9c19cfac0b37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731187093235 2024-11-09T21:18:17,346 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04f7cdbf23af4015875c63205f9e1ad0, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731187095305 2024-11-09T21:18:17,346 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 38d41d9a457e43289cb7d199dd389d80, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731187095305 2024-11-09T21:18:17,361 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:17,362 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#B#compaction#504 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:17,362 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/2e22423180da4dfa8970c7161896dc05 is 50, key is test_row_0/B:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:17,369 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411097e134f87ad144ef28a01ec4232b222bf_172f8346f592a41a81f031b40097d85b store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:17,371 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411097e134f87ad144ef28a01ec4232b222bf_172f8346f592a41a81f031b40097d85b, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:17,371 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411097e134f87ad144ef28a01ec4232b222bf_172f8346f592a41a81f031b40097d85b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:17,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-09T21:18:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:17,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:17,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742414_1590 (size=4469) 2024-11-09T21:18:17,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742415_1591 (size=12359) 2024-11-09T21:18:17,437 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/2e22423180da4dfa8970c7161896dc05 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2e22423180da4dfa8970c7161896dc05 2024-11-09T21:18:17,441 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/B of 172f8346f592a41a81f031b40097d85b into 2e22423180da4dfa8970c7161896dc05(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:17,441 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:17,441 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/B, priority=13, startTime=1731187097344; duration=0sec 2024-11-09T21:18:17,442 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:17,442 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:B 2024-11-09T21:18:17,442 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:17,442 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:17,443 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/C is initiating minor compaction (all files) 2024-11-09T21:18:17,443 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/C in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,443 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/1611186e516040f99aba56113ebf92b4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/af787d025fbd4a049cef3f4baa7febdb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/30ce283e1c2547ac8e814c4156cbbe23] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=35.4 K 2024-11-09T21:18:17,443 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 1611186e516040f99aba56113ebf92b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731187092593 2024-11-09T21:18:17,444 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting af787d025fbd4a049cef3f4baa7febdb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1731187093235 2024-11-09T21:18:17,444 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 30ce283e1c2547ac8e814c4156cbbe23, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731187095305 2024-11-09T21:18:17,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109551af29681d444aeadff4834464fce0c_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187097408/Put/seqid=0 2024-11-09T21:18:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187157441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,447 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187157441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187157443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187157445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,464 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#C#compaction#507 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:17,465 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/73fbf675da4f4cb5b291fdb02c8c65b1 is 50, key is test_row_0/C:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:17,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742416_1592 (size=14794) 2024-11-09T21:18:17,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742417_1593 (size=12359) 2024-11-09T21:18:17,557 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187157556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187157556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187157556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187157556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,558 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:17,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:17,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,564 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:17,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187157766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187157767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,768 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187157767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187157767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,829 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#A#compaction#505 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:17,829 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4aea51f6db6446ec9fd7d64d8bc17432 is 175, key is test_row_0/A:col10/1731187095668/Put/seqid=0 2024-11-09T21:18:17,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742418_1594 (size=31313) 2024-11-09T21:18:17,837 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4aea51f6db6446ec9fd7d64d8bc17432 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4aea51f6db6446ec9fd7d64d8bc17432 2024-11-09T21:18:17,842 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/A of 172f8346f592a41a81f031b40097d85b into 4aea51f6db6446ec9fd7d64d8bc17432(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:17,842 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:17,842 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/A, priority=13, startTime=1731187097342; duration=0sec 2024-11-09T21:18:17,842 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:17,842 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:A 2024-11-09T21:18:17,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:17,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:17,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:17,886 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:17,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109551af29681d444aeadff4834464fce0c_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109551af29681d444aeadff4834464fce0c_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:17,890 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/7a22593703e44c1191d5ac2d3399d682, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:17,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/7a22593703e44c1191d5ac2d3399d682 is 175, key is test_row_0/A:col10/1731187097408/Put/seqid=0 2024-11-09T21:18:17,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:17,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187157896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:17,898 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2148 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:17,898 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:17,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742419_1595 (size=39749) 2024-11-09T21:18:17,921 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/73fbf675da4f4cb5b291fdb02c8c65b1 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/73fbf675da4f4cb5b291fdb02c8c65b1 2024-11-09T21:18:17,925 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/C of 172f8346f592a41a81f031b40097d85b into 73fbf675da4f4cb5b291fdb02c8c65b1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:17,925 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:17,925 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/C, priority=13, startTime=1731187097345; duration=0sec 2024-11-09T21:18:17,925 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:17,925 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:C 2024-11-09T21:18:18,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:18,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187158076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187158076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187158076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187158076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,306 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/7a22593703e44c1191d5ac2d3399d682 2024-11-09T21:18:18,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/64e1cffec98a4698bcee81f8314813b0 is 50, key is test_row_0/B:col10/1731187097408/Put/seqid=0 2024-11-09T21:18:18,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742420_1596 (size=12151) 2024-11-09T21:18:18,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:18,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187158586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187158586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187158587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,589 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187158587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:18,590 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:18,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:18,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/64e1cffec98a4698bcee81f8314813b0 2024-11-09T21:18:18,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/458ae283c4ca4eee8ebc1d8dcbb0edf2 is 50, key is test_row_0/C:col10/1731187097408/Put/seqid=0 2024-11-09T21:18:18,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742421_1597 (size=12151) 2024-11-09T21:18:18,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/458ae283c4ca4eee8ebc1d8dcbb0edf2 2024-11-09T21:18:18,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/7a22593703e44c1191d5ac2d3399d682 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682 2024-11-09T21:18:18,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682, entries=200, sequenceid=152, filesize=38.8 K 2024-11-09T21:18:18,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/64e1cffec98a4698bcee81f8314813b0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/64e1cffec98a4698bcee81f8314813b0 2024-11-09T21:18:18,736 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/64e1cffec98a4698bcee81f8314813b0, entries=150, sequenceid=152, filesize=11.9 K 2024-11-09T21:18:18,737 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/458ae283c4ca4eee8ebc1d8dcbb0edf2 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/458ae283c4ca4eee8ebc1d8dcbb0edf2 2024-11-09T21:18:18,745 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/458ae283c4ca4eee8ebc1d8dcbb0edf2, entries=150, sequenceid=152, filesize=11.9 K 2024-11-09T21:18:18,746 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 172f8346f592a41a81f031b40097d85b in 1336ms, sequenceid=152, compaction requested=false 2024-11-09T21:18:18,746 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:18,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:18,785 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:18,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:18,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a30aa7f91fbe4ed28682ddef08bd2255_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187097428/Put/seqid=0 2024-11-09T21:18:18,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742422_1598 (size=12304) 2024-11-09T21:18:19,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:19,235 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109a30aa7f91fbe4ed28682ddef08bd2255_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a30aa7f91fbe4ed28682ddef08bd2255_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:19,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4181cd058f1b44eb932a2c1b718d4fd7, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:19,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4181cd058f1b44eb932a2c1b718d4fd7 is 175, key is test_row_0/A:col10/1731187097428/Put/seqid=0 2024-11-09T21:18:19,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742423_1599 (size=31105) 2024-11-09T21:18:19,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:19,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:19,625 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187159621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187159622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187159623, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187159624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,628 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,654 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=168, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4181cd058f1b44eb932a2c1b718d4fd7 2024-11-09T21:18:19,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/7d28d84f53444e18a2cbb16a3361dde6 is 50, key is test_row_0/B:col10/1731187097428/Put/seqid=0 2024-11-09T21:18:19,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742424_1600 (size=12151) 2024-11-09T21:18:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187159736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187159737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187159737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187159737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,739 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,740 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187159917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,920 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187159946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,948 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187159946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187159946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:19,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:19,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187159946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,063 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/7d28d84f53444e18a2cbb16a3361dde6 2024-11-09T21:18:20,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ded864afcbfd4f10a21e41045354c90a is 50, key is test_row_0/C:col10/1731187097428/Put/seqid=0 2024-11-09T21:18:20,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742425_1601 (size=12151) 2024-11-09T21:18:20,072 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ded864afcbfd4f10a21e41045354c90a 2024-11-09T21:18:20,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4181cd058f1b44eb932a2c1b718d4fd7 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7 2024-11-09T21:18:20,079 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7, entries=150, sequenceid=168, filesize=30.4 K 2024-11-09T21:18:20,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/7d28d84f53444e18a2cbb16a3361dde6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7d28d84f53444e18a2cbb16a3361dde6 2024-11-09T21:18:20,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7d28d84f53444e18a2cbb16a3361dde6, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T21:18:20,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ded864afcbfd4f10a21e41045354c90a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ded864afcbfd4f10a21e41045354c90a 2024-11-09T21:18:20,085 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ded864afcbfd4f10a21e41045354c90a, entries=150, sequenceid=168, filesize=11.9 K 2024-11-09T21:18:20,086 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 172f8346f592a41a81f031b40097d85b in 1301ms, sequenceid=168, compaction requested=true 2024-11-09T21:18:20,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:20,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:20,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-09T21:18:20,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-09T21:18:20,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-09T21:18:20,088 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.5380 sec 2024-11-09T21:18:20,090 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 4.5420 sec 2024-11-09T21:18:20,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:20,260 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-09T21:18:20,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:20,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:20,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:20,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:20,260 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:20,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:20,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dafd10bc8656484e9758a87e319cf36f_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:20,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742426_1602 (size=17284) 2024-11-09T21:18:20,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187160278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187160278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187160282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187160285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187160396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187160397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187160397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187160397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,402 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187160606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187160606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187160607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187160607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,674 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:20,676 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109dafd10bc8656484e9758a87e319cf36f_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dafd10bc8656484e9758a87e319cf36f_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:20,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/fd874d505cef483b9dbc711a226089e4, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:20,677 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/fd874d505cef483b9dbc711a226089e4 is 175, key is test_row_0/A:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:20,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742427_1603 (size=48389) 2024-11-09T21:18:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187160917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,918 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187160917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187160917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:20,919 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:20,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187160917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:20,920 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,081 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/fd874d505cef483b9dbc711a226089e4 2024-11-09T21:18:21,086 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/4ae9efc50140407f808a8f1538af2ffe is 50, key is test_row_0/B:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:21,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742428_1604 (size=12151) 2024-11-09T21:18:21,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/4ae9efc50140407f808a8f1538af2ffe 2024-11-09T21:18:21,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/5d02974c946745d7986d1424d47f656a is 50, key is test_row_0/C:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:21,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742429_1605 (size=12151) 2024-11-09T21:18:21,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/5d02974c946745d7986d1424d47f656a 2024-11-09T21:18:21,118 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/fd874d505cef483b9dbc711a226089e4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4 2024-11-09T21:18:21,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4, entries=250, sequenceid=189, filesize=47.3 K 2024-11-09T21:18:21,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/4ae9efc50140407f808a8f1538af2ffe as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/4ae9efc50140407f808a8f1538af2ffe 2024-11-09T21:18:21,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/4ae9efc50140407f808a8f1538af2ffe, entries=150, sequenceid=189, filesize=11.9 K 2024-11-09T21:18:21,128 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/5d02974c946745d7986d1424d47f656a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/5d02974c946745d7986d1424d47f656a 2024-11-09T21:18:21,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/5d02974c946745d7986d1424d47f656a, entries=150, sequenceid=189, filesize=11.9 K 2024-11-09T21:18:21,132 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 172f8346f592a41a81f031b40097d85b in 872ms, sequenceid=189, compaction requested=true 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:18:21,132 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:21,132 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:18:21,132 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:21,133 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:18:21,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 150556 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:18:21,133 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/B is initiating minor compaction (all files) 2024-11-09T21:18:21,133 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/A is initiating minor compaction (all files) 2024-11-09T21:18:21,133 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/B in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:21,133 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2e22423180da4dfa8970c7161896dc05, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/64e1cffec98a4698bcee81f8314813b0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7d28d84f53444e18a2cbb16a3361dde6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/4ae9efc50140407f808a8f1538af2ffe] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=47.7 K 2024-11-09T21:18:21,134 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/A in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:21,134 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4aea51f6db6446ec9fd7d64d8bc17432, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=147.0 K 2024-11-09T21:18:21,134 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:21,134 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4aea51f6db6446ec9fd7d64d8bc17432, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4] 2024-11-09T21:18:21,134 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e22423180da4dfa8970c7161896dc05, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731187095305 2024-11-09T21:18:21,135 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4aea51f6db6446ec9fd7d64d8bc17432, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731187095305 2024-11-09T21:18:21,135 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 64e1cffec98a4698bcee81f8314813b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731187095742 2024-11-09T21:18:21,136 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a22593703e44c1191d5ac2d3399d682, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731187095736 2024-11-09T21:18:21,136 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d28d84f53444e18a2cbb16a3361dde6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731187097428 2024-11-09T21:18:21,136 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4181cd058f1b44eb932a2c1b718d4fd7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731187097428 2024-11-09T21:18:21,136 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ae9efc50140407f808a8f1538af2ffe, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731187099623 2024-11-09T21:18:21,136 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd874d505cef483b9dbc711a226089e4, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731187099615 2024-11-09T21:18:21,146 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:21,149 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109144c425d5f204ac5ac91558817c17d3e_172f8346f592a41a81f031b40097d85b store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:21,150 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#B#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:21,151 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/ebeb208e3c31458aa9672228df8e5c14 is 50, key is test_row_0/B:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:21,152 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109144c425d5f204ac5ac91558817c17d3e_172f8346f592a41a81f031b40097d85b, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:21,152 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109144c425d5f204ac5ac91558817c17d3e_172f8346f592a41a81f031b40097d85b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:21,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742430_1606 (size=12595) 2024-11-09T21:18:21,170 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/ebeb208e3c31458aa9672228df8e5c14 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ebeb208e3c31458aa9672228df8e5c14 2024-11-09T21:18:21,178 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 172f8346f592a41a81f031b40097d85b/B of 172f8346f592a41a81f031b40097d85b into ebeb208e3c31458aa9672228df8e5c14(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:21,178 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:21,178 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/B, priority=12, startTime=1731187101132; duration=0sec 2024-11-09T21:18:21,178 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:21,178 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:B 2024-11-09T21:18:21,178 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:18:21,179 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48812 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:18:21,179 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/C is initiating minor compaction (all files) 2024-11-09T21:18:21,179 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/C in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:21,179 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/73fbf675da4f4cb5b291fdb02c8c65b1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/458ae283c4ca4eee8ebc1d8dcbb0edf2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ded864afcbfd4f10a21e41045354c90a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/5d02974c946745d7986d1424d47f656a] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=47.7 K 2024-11-09T21:18:21,180 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 73fbf675da4f4cb5b291fdb02c8c65b1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731187095305 2024-11-09T21:18:21,180 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 458ae283c4ca4eee8ebc1d8dcbb0edf2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731187095742 2024-11-09T21:18:21,181 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ded864afcbfd4f10a21e41045354c90a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=168, earliestPutTs=1731187097428 2024-11-09T21:18:21,181 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d02974c946745d7986d1424d47f656a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731187099623 2024-11-09T21:18:21,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742431_1607 (size=4469) 2024-11-09T21:18:21,187 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#A#compaction#516 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:21,188 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/b9642ffa23fe40d19bed71f38e8b55e0 is 175, key is test_row_0/A:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:21,192 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:21,193 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/caca1021a6044c11aa51b7bf20dfa4a6 is 50, key is test_row_0/C:col10/1731187100258/Put/seqid=0 2024-11-09T21:18:21,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742432_1608 (size=31549) 2024-11-09T21:18:21,220 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/b9642ffa23fe40d19bed71f38e8b55e0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/b9642ffa23fe40d19bed71f38e8b55e0 2024-11-09T21:18:21,224 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 172f8346f592a41a81f031b40097d85b/A of 172f8346f592a41a81f031b40097d85b into b9642ffa23fe40d19bed71f38e8b55e0(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:21,224 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:21,224 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/A, priority=12, startTime=1731187101132; duration=0sec 2024-11-09T21:18:21,224 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:21,224 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:A 2024-11-09T21:18:21,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742433_1609 (size=12595) 2024-11-09T21:18:21,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:21,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:18:21,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:21,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:21,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:21,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:21,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:21,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:21,435 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109decca35739324cdba44f7332d96e32ef_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187100280/Put/seqid=0 2024-11-09T21:18:21,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742434_1610 (size=12304) 2024-11-09T21:18:21,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:21,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109decca35739324cdba44f7332d96e32ef_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109decca35739324cdba44f7332d96e32ef_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:21,441 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e585e8a8266c42a0940bf83226080b21, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:21,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e585e8a8266c42a0940bf83226080b21 is 175, key is test_row_0/A:col10/1731187100280/Put/seqid=0 2024-11-09T21:18:21,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742435_1611 (size=31105) 2024-11-09T21:18:21,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187161449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,456 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187161452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,458 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187161453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187161455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,569 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187161566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187161566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187161566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187161566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,631 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/caca1021a6044c11aa51b7bf20dfa4a6 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/caca1021a6044c11aa51b7bf20dfa4a6 2024-11-09T21:18:21,635 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 172f8346f592a41a81f031b40097d85b/C of 172f8346f592a41a81f031b40097d85b into caca1021a6044c11aa51b7bf20dfa4a6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:21,635 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:21,635 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/C, priority=12, startTime=1731187101132; duration=0sec 2024-11-09T21:18:21,635 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:21,635 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:C 2024-11-09T21:18:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187161776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187161776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187161776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187161776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:21,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:21,849 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=204, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e585e8a8266c42a0940bf83226080b21 2024-11-09T21:18:21,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/9894ee1ed5a54e0d896f6795a70b3c67 is 50, key is test_row_0/B:col10/1731187100280/Put/seqid=0 2024-11-09T21:18:21,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742436_1612 (size=12151) 2024-11-09T21:18:21,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/9894ee1ed5a54e0d896f6795a70b3c67 2024-11-09T21:18:21,862 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/8bd0aab4badc450faa769eff1b9c125d is 50, key is test_row_0/C:col10/1731187100280/Put/seqid=0 2024-11-09T21:18:21,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742437_1613 (size=12151) 2024-11-09T21:18:22,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187162086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187162086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,088 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187162087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187162087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,090 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/8bd0aab4badc450faa769eff1b9c125d 2024-11-09T21:18:22,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e585e8a8266c42a0940bf83226080b21 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21 2024-11-09T21:18:22,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21, entries=150, sequenceid=204, filesize=30.4 K 2024-11-09T21:18:22,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/9894ee1ed5a54e0d896f6795a70b3c67 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/9894ee1ed5a54e0d896f6795a70b3c67 2024-11-09T21:18:22,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/9894ee1ed5a54e0d896f6795a70b3c67, entries=150, sequenceid=204, filesize=11.9 K 2024-11-09T21:18:22,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/8bd0aab4badc450faa769eff1b9c125d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8bd0aab4badc450faa769eff1b9c125d 2024-11-09T21:18:22,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8bd0aab4badc450faa769eff1b9c125d, entries=150, sequenceid=204, filesize=11.9 K 2024-11-09T21:18:22,278 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 172f8346f592a41a81f031b40097d85b in 849ms, sequenceid=204, compaction requested=false 2024-11-09T21:18:22,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:22,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:22,600 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-09T21:18:22,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:22,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:22,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:22,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:22,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:22,601 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:22,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b740f03531d84fcd8db1fc37371f95e3_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:22,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742438_1614 (size=14794) 2024-11-09T21:18:22,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187162610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,616 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187162615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,617 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187162615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187162616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187162726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187162726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,729 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187162726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187162726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187162936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,939 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187162936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,940 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187162936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:22,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:22,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187162940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:22,945 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,014 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:23,017 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109b740f03531d84fcd8db1fc37371f95e3_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b740f03531d84fcd8db1fc37371f95e3_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:23,017 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/07e445c3aa164b6b835ddba707cfebd4, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:23,018 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/07e445c3aa164b6b835ddba707cfebd4 is 175, key is test_row_0/A:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:23,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742439_1615 (size=39749) 2024-11-09T21:18:23,022 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=227, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/07e445c3aa164b6b835ddba707cfebd4 2024-11-09T21:18:23,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/096255bd859c4a71b03264195d5ff50f is 50, key is test_row_0/B:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:23,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742440_1616 (size=12151) 2024-11-09T21:18:23,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187163247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187163247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,249 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187163247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,249 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187163256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/096255bd859c4a71b03264195d5ff50f 2024-11-09T21:18:23,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/84b591699a7d47aa92af19fc830a2acf is 50, key is test_row_0/C:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:23,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742441_1617 (size=12151) 2024-11-09T21:18:23,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=227 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/84b591699a7d47aa92af19fc830a2acf 2024-11-09T21:18:23,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/07e445c3aa164b6b835ddba707cfebd4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4 2024-11-09T21:18:23,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4, entries=200, sequenceid=227, filesize=38.8 K 2024-11-09T21:18:23,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/096255bd859c4a71b03264195d5ff50f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/096255bd859c4a71b03264195d5ff50f 2024-11-09T21:18:23,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/096255bd859c4a71b03264195d5ff50f, entries=150, sequenceid=227, filesize=11.9 K 2024-11-09T21:18:23,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/84b591699a7d47aa92af19fc830a2acf as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/84b591699a7d47aa92af19fc830a2acf 2024-11-09T21:18:23,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/84b591699a7d47aa92af19fc830a2acf, entries=150, sequenceid=227, filesize=11.9 K 2024-11-09T21:18:23,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=46.96 KB/48090 for 172f8346f592a41a81f031b40097d85b in 879ms, sequenceid=227, compaction requested=true 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:23,479 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:23,479 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:18:23,479 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:23,481 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:23,481 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:23,481 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/B is initiating minor compaction (all files) 2024-11-09T21:18:23,481 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/A is initiating minor compaction (all files) 2024-11-09T21:18:23,481 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/B in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:23,482 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/A in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:23,482 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ebeb208e3c31458aa9672228df8e5c14, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/9894ee1ed5a54e0d896f6795a70b3c67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/096255bd859c4a71b03264195d5ff50f] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=36.0 K 2024-11-09T21:18:23,482 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/b9642ffa23fe40d19bed71f38e8b55e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=100.0 K 2024-11-09T21:18:23,482 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:23,482 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/b9642ffa23fe40d19bed71f38e8b55e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4] 2024-11-09T21:18:23,482 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ebeb208e3c31458aa9672228df8e5c14, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731187099623 2024-11-09T21:18:23,482 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9642ffa23fe40d19bed71f38e8b55e0, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731187099623 2024-11-09T21:18:23,482 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 9894ee1ed5a54e0d896f6795a70b3c67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731187100280 2024-11-09T21:18:23,482 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e585e8a8266c42a0940bf83226080b21, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731187100280 2024-11-09T21:18:23,483 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07e445c3aa164b6b835ddba707cfebd4, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731187101451 2024-11-09T21:18:23,483 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 096255bd859c4a71b03264195d5ff50f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731187101451 2024-11-09T21:18:23,491 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:23,494 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#B#compaction#526 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:23,495 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/35378fc3122746adbdbf4a3e814c3bf4 is 50, key is test_row_0/B:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:23,505 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109412bb4e9a7bc4c5a81782e18f556a13b_172f8346f592a41a81f031b40097d85b store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:23,507 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109412bb4e9a7bc4c5a81782e18f556a13b_172f8346f592a41a81f031b40097d85b, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:23,507 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109412bb4e9a7bc4c5a81782e18f556a13b_172f8346f592a41a81f031b40097d85b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:23,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742442_1618 (size=12697) 2024-11-09T21:18:23,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742443_1619 (size=4469) 2024-11-09T21:18:23,517 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#A#compaction#525 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:23,518 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/8d9e0321696247f7be1d82574b007172 is 175, key is test_row_0/A:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:23,521 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/35378fc3122746adbdbf4a3e814c3bf4 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/35378fc3122746adbdbf4a3e814c3bf4 2024-11-09T21:18:23,526 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/B of 172f8346f592a41a81f031b40097d85b into 35378fc3122746adbdbf4a3e814c3bf4(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:23,527 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:23,527 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/B, priority=13, startTime=1731187103479; duration=0sec 2024-11-09T21:18:23,527 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:23,527 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:B 2024-11-09T21:18:23,527 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-09T21:18:23,527 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-09T21:18:23,527 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/C is initiating minor compaction (all files) 2024-11-09T21:18:23,528 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/C in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:23,528 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/caca1021a6044c11aa51b7bf20dfa4a6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8bd0aab4badc450faa769eff1b9c125d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/84b591699a7d47aa92af19fc830a2acf] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=36.0 K 2024-11-09T21:18:23,528 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting caca1021a6044c11aa51b7bf20dfa4a6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731187099623 2024-11-09T21:18:23,528 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd0aab4badc450faa769eff1b9c125d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731187100280 2024-11-09T21:18:23,529 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 84b591699a7d47aa92af19fc830a2acf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731187101451 2024-11-09T21:18:23,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742444_1620 (size=31651) 2024-11-09T21:18:23,534 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#C#compaction#527 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:23,535 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/3001b776a23945f38058021d70657d6f is 50, key is test_row_0/C:col10/1731187101451/Put/seqid=0 2024-11-09T21:18:23,535 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/8d9e0321696247f7be1d82574b007172 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/8d9e0321696247f7be1d82574b007172 2024-11-09T21:18:23,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742445_1621 (size=12697) 2024-11-09T21:18:23,543 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/A of 172f8346f592a41a81f031b40097d85b into 8d9e0321696247f7be1d82574b007172(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:23,543 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:23,543 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/A, priority=13, startTime=1731187103479; duration=0sec 2024-11-09T21:18:23,543 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:23,543 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:A 2024-11-09T21:18:23,545 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/3001b776a23945f38058021d70657d6f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/3001b776a23945f38058021d70657d6f 2024-11-09T21:18:23,548 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 172f8346f592a41a81f031b40097d85b/C of 172f8346f592a41a81f031b40097d85b into 3001b776a23945f38058021d70657d6f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:23,548 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:23,548 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/C, priority=13, startTime=1731187103479; duration=0sec 2024-11-09T21:18:23,548 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:23,548 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:C 2024-11-09T21:18:23,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-09T21:18:23,746 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:23,747 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-09T21:18:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-09T21:18:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:23,748 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-09T21:18:23,749 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-09T21:18:23,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-09T21:18:23,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:23,758 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-09T21:18:23,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:23,758 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:23,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:23,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:23,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:23,759 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:23,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099121ca2daea54da7a9c6ff93a5eea085_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187103757/Put/seqid=0 2024-11-09T21:18:23,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742446_1622 (size=12304) 2024-11-09T21:18:23,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187163782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,787 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187163783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187103775, max timestamp=1731187103785Num uniques 859; 2024-11-09T21:18:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:18:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187103775, max timestamp=1731187103785Num uniques 859; 2024-11-09T21:18:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:18:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731187103775, max timestamp=1731187103785Num uniques 859; 2024-11-09T21:18:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,788 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/9514d6d5e5be:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-09T21:18:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,788 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,832 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187163826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187163828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,835 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:23,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187163897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187163897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,901 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,902 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:23,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:23,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:23,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:23,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187163936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,940 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187163946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:23,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:23,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33170 deadline: 1731187163956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:23,959 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:23,959 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:24,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187164107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187164107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187164146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187164156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,161 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,180 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:24,183 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411099121ca2daea54da7a9c6ff93a5eea085_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099121ca2daea54da7a9c6ff93a5eea085_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:24,183 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/ff1a7f4a387044148d1b61258104e011, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:24,184 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/ff1a7f4a387044148d1b61258104e011 is 175, key is test_row_0/A:col10/1731187103757/Put/seqid=0 2024-11-09T21:18:24,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742447_1623 (size=31105) 2024-11-09T21:18:24,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187164417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,419 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187164417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,420 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187164456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187164466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,470 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,513 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,588 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/ff1a7f4a387044148d1b61258104e011 2024-11-09T21:18:24,594 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/ad6e58f1b4ac439a8708fedd40dad45f is 50, key is test_row_0/B:col10/1731187103757/Put/seqid=0 2024-11-09T21:18:24,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742448_1624 (size=12151) 2024-11-09T21:18:24,665 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,666 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:24,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187164926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187164926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187164967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,970 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:24,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:24,972 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:24,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:24,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187164977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:24,978 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:25,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/ad6e58f1b4ac439a8708fedd40dad45f 2024-11-09T21:18:25,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/bbc6ce9602884934b5f04ce87fb10e52 is 50, key is test_row_0/C:col10/1731187103757/Put/seqid=0 2024-11-09T21:18:25,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742449_1625 (size=12151) 2024-11-09T21:18:25,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:25,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,435 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:25,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T21:18:25,472 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/bbc6ce9602884934b5f04ce87fb10e52 2024-11-09T21:18:25,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/ff1a7f4a387044148d1b61258104e011 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011 2024-11-09T21:18:25,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011, entries=150, sequenceid=241, filesize=30.4 K 2024-11-09T21:18:25,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/ad6e58f1b4ac439a8708fedd40dad45f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ad6e58f1b4ac439a8708fedd40dad45f 2024-11-09T21:18:25,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ad6e58f1b4ac439a8708fedd40dad45f, entries=150, sequenceid=241, filesize=11.9 K 2024-11-09T21:18:25,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/bbc6ce9602884934b5f04ce87fb10e52 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/bbc6ce9602884934b5f04ce87fb10e52 2024-11-09T21:18:25,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/bbc6ce9602884934b5f04ce87fb10e52, entries=150, sequenceid=241, filesize=11.9 K 2024-11-09T21:18:25,496 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 172f8346f592a41a81f031b40097d85b in 1738ms, sequenceid=241, compaction requested=false 2024-11-09T21:18:25,496 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:25,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=38413 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:25,591 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:25,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-09T21:18:25,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109689e6d13533e4826b96e397a48a8b5ab_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187103818/Put/seqid=0 2024-11-09T21:18:25,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742450_1626 (size=12454) 2024-11-09T21:18:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:25,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. as already flushing 2024-11-09T21:18:25,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187165958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:25,964 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:25,965 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187165964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:25,969 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:25,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33122 deadline: 1731187165976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:25,979 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_2' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2153 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:25,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:25,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33184 deadline: 1731187165996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:25,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:25,999 WARN [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRpcRetryingCaller(177): Call to 9514d6d5e5be,38413,1731186934857 for 'test_row_1' in 172f8346f592a41a81f031b40097d85b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor44.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-09T21:18:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,002 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109689e6d13533e4826b96e397a48a8b5ab_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109689e6d13533e4826b96e397a48a8b5ab_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6825c6841d4d4136b3b85ed83ea32f5d, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:26,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6825c6841d4d4136b3b85ed83ea32f5d is 175, key is test_row_0/A:col10/1731187103818/Put/seqid=0 2024-11-09T21:18:26,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742451_1627 (size=31255) 2024-11-09T21:18:26,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187166066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187166076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,080 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187166276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,278 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187166286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,407 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6825c6841d4d4136b3b85ed83ea32f5d 2024-11-09T21:18:26,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/e88771761cd44804b311bf3d481925ef is 50, key is test_row_0/B:col10/1731187103818/Put/seqid=0 2024-11-09T21:18:26,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742452_1628 (size=12301) 2024-11-09T21:18:26,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:26,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187166586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:26,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187166596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:26,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:26,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/e88771761cd44804b311bf3d481925ef 2024-11-09T21:18:26,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/e1b7587a6f834557bdcebe1627290c3a is 50, key is test_row_0/C:col10/1731187103818/Put/seqid=0 2024-11-09T21:18:26,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742453_1629 (size=12301) 2024-11-09T21:18:27,061 INFO [Thread-2477 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2477. 2024-11-09T21:18:27,061 DEBUG [Thread-2477 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:27,061 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,061 DEBUG [Thread-2477 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,061 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:27,065 INFO [Thread-2469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2469. 2024-11-09T21:18:27,065 DEBUG [Thread-2469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:27,065 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,066 INFO [Thread-2471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2471. 2024-11-09T21:18:27,066 DEBUG [Thread-2469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,066 DEBUG [Thread-2471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:27,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:27,066 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,066 DEBUG [Thread-2471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,066 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:27,067 INFO [Thread-2475 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2475. 2024-11-09T21:18:27,067 DEBUG [Thread-2475 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:27,067 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,068 DEBUG [Thread-2475 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,068 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:27,072 INFO [Thread-2473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2473. 2024-11-09T21:18:27,072 DEBUG [Thread-2473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:27,072 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,072 DEBUG [Thread-2473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,072 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:27,096 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33162 deadline: 1731187167096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:27,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:27,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:27,097 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:27,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T21:18:27,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:33194 deadline: 1731187167106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 , the old value is region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-09T21:18:27,107 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., hostname=9514d6d5e5be,38413,1731186934857, seqNum=5 because the exception is null or not the one we care about 2024-11-09T21:18:27,230 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/e1b7587a6f834557bdcebe1627290c3a 2024-11-09T21:18:27,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/6825c6841d4d4136b3b85ed83ea32f5d as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d 2024-11-09T21:18:27,236 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d, entries=150, sequenceid=276, filesize=30.5 K 2024-11-09T21:18:27,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/e88771761cd44804b311bf3d481925ef as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/e88771761cd44804b311bf3d481925ef 2024-11-09T21:18:27,240 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/e88771761cd44804b311bf3d481925ef, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T21:18:27,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/e1b7587a6f834557bdcebe1627290c3a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/e1b7587a6f834557bdcebe1627290c3a 2024-11-09T21:18:27,243 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/e1b7587a6f834557bdcebe1627290c3a, entries=150, sequenceid=276, filesize=12.0 K 2024-11-09T21:18:27,243 INFO [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 172f8346f592a41a81f031b40097d85b in 1652ms, sequenceid=276, compaction requested=true 2024-11-09T21:18:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:27,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9514d6d5e5be:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-09T21:18:27,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-09T21:18:27,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-09T21:18:27,246 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4960 sec 2024-11-09T21:18:27,247 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 3.4990 sec 2024-11-09T21:18:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-09T21:18:27,917 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38413 {}] regionserver.HRegion(8855): Flush requested on 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:27,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-09T21:18:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:27,998 INFO [Thread-2466 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2466. 2024-11-09T21:18:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:27,998 DEBUG [Thread-2466 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:27,998 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:27,998 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:27,998 DEBUG [Thread-2466 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:27,998 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:28,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094badf7ca72ec41edbbc8f3399c55baa5_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:28,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742454_1630 (size=12454) 2024-11-09T21:18:28,008 INFO [Thread-2462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2462. 2024-11-09T21:18:28,008 DEBUG [Thread-2462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:28,008 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:28,008 DEBUG [Thread-2462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:28,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:28,108 INFO [Thread-2460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2460. 2024-11-09T21:18:28,108 DEBUG [Thread-2460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:28,108 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:28,108 DEBUG [Thread-2460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:28,108 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:28,117 INFO [Thread-2464 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2464. 2024-11-09T21:18:28,117 DEBUG [Thread-2464 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:28,117 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:28,117 DEBUG [Thread-2464 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:28,117 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:28,405 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:28,408 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411094badf7ca72ec41edbbc8f3399c55baa5_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094badf7ca72ec41edbbc8f3399c55baa5_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:28,409 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e1dba02605f14b1e91fc7fd1550306b3, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:28,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e1dba02605f14b1e91fc7fd1550306b3 is 175, key is test_row_0/A:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:28,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742455_1631 (size=31255) 2024-11-09T21:18:28,813 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=288, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e1dba02605f14b1e91fc7fd1550306b3 2024-11-09T21:18:28,818 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/f12eadff7211465c815bc0873dbecbde is 50, key is test_row_0/B:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:28,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742456_1632 (size=12301) 2024-11-09T21:18:29,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/f12eadff7211465c815bc0873dbecbde 2024-11-09T21:18:29,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/fa7d72b002ca480182f044a3977eb08c is 50, key is test_row_0/C:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:29,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742457_1633 (size=12301) 2024-11-09T21:18:29,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=288 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/fa7d72b002ca480182f044a3977eb08c 2024-11-09T21:18:29,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/e1dba02605f14b1e91fc7fd1550306b3 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3 2024-11-09T21:18:29,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3, entries=150, sequenceid=288, filesize=30.5 K 2024-11-09T21:18:29,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/f12eadff7211465c815bc0873dbecbde as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/f12eadff7211465c815bc0873dbecbde 2024-11-09T21:18:29,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/f12eadff7211465c815bc0873dbecbde, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T21:18:29,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/fa7d72b002ca480182f044a3977eb08c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/fa7d72b002ca480182f044a3977eb08c 2024-11-09T21:18:29,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/fa7d72b002ca480182f044a3977eb08c, entries=150, sequenceid=288, filesize=12.0 K 2024-11-09T21:18:29,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 172f8346f592a41a81f031b40097d85b in 1645ms, sequenceid=288, compaction requested=true 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:A, priority=-2147483648, current under compaction store size is 1 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:B, priority=-2147483648, current under compaction store size is 2 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:29,642 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 172f8346f592a41a81f031b40097d85b:C, priority=-2147483648, current under compaction store size is 3 2024-11-09T21:18:29,642 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:18:29,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:29,643 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:18:29,643 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:18:29,643 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/A is initiating minor compaction (all files) 2024-11-09T21:18:29,643 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/B is initiating minor compaction (all files) 2024-11-09T21:18:29,643 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/A in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:29,643 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/B in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:29,643 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/8d9e0321696247f7be1d82574b007172, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=122.3 K 2024-11-09T21:18:29,643 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/35378fc3122746adbdbf4a3e814c3bf4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ad6e58f1b4ac439a8708fedd40dad45f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/e88771761cd44804b311bf3d481925ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/f12eadff7211465c815bc0873dbecbde] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=48.3 K 2024-11-09T21:18:29,643 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. files: [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/8d9e0321696247f7be1d82574b007172, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3] 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 35378fc3122746adbdbf4a3e814c3bf4, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731187101451 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9e0321696247f7be1d82574b007172, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731187101451 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting ad6e58f1b4ac439a8708fedd40dad45f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731187102604 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting ff1a7f4a387044148d1b61258104e011, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731187102604 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e88771761cd44804b311bf3d481925ef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731187103818 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6825c6841d4d4136b3b85ed83ea32f5d, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731187103818 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting f12eadff7211465c815bc0873dbecbde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731187105943 2024-11-09T21:18:29,644 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1dba02605f14b1e91fc7fd1550306b3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731187105943 2024-11-09T21:18:29,649 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:29,650 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241109c7e17bf513af476ba210d66543ff3903_172f8346f592a41a81f031b40097d85b store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:29,652 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#B#compaction#537 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:29,652 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/7901b4afe23c4955a2d8cd0c39dded4b is 50, key is test_row_0/B:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:29,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742458_1634 (size=12983) 2024-11-09T21:18:29,659 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241109c7e17bf513af476ba210d66543ff3903_172f8346f592a41a81f031b40097d85b, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:29,659 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241109c7e17bf513af476ba210d66543ff3903_172f8346f592a41a81f031b40097d85b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:29,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742459_1635 (size=4469) 2024-11-09T21:18:30,067 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#A#compaction#538 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:30,068 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4d46d0fb906d4a8b99071718ce876227 is 175, key is test_row_0/A:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:30,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742460_1636 (size=31937) 2024-11-09T21:18:30,123 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/7901b4afe23c4955a2d8cd0c39dded4b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7901b4afe23c4955a2d8cd0c39dded4b 2024-11-09T21:18:30,126 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 172f8346f592a41a81f031b40097d85b/B of 172f8346f592a41a81f031b40097d85b into 7901b4afe23c4955a2d8cd0c39dded4b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:30,126 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:30,126 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/B, priority=12, startTime=1731187109642; duration=0sec 2024-11-09T21:18:30,126 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-09T21:18:30,126 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:B 2024-11-09T21:18:30,126 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-09T21:18:30,127 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49450 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-09T21:18:30,127 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1541): 172f8346f592a41a81f031b40097d85b/C is initiating minor compaction (all files) 2024-11-09T21:18:30,127 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 172f8346f592a41a81f031b40097d85b/C in TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:30,127 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/3001b776a23945f38058021d70657d6f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/bbc6ce9602884934b5f04ce87fb10e52, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/e1b7587a6f834557bdcebe1627290c3a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/fa7d72b002ca480182f044a3977eb08c] into tmpdir=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp, totalSize=48.3 K 2024-11-09T21:18:30,127 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting 3001b776a23945f38058021d70657d6f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=227, earliestPutTs=1731187101451 2024-11-09T21:18:30,128 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting bbc6ce9602884934b5f04ce87fb10e52, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731187102604 2024-11-09T21:18:30,128 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting e1b7587a6f834557bdcebe1627290c3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1731187103818 2024-11-09T21:18:30,128 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] compactions.Compactor(225): Compacting fa7d72b002ca480182f044a3977eb08c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=288, earliestPutTs=1731187105943 2024-11-09T21:18:30,135 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 172f8346f592a41a81f031b40097d85b#C#compaction#539 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-09T21:18:30,136 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ce1e1e1df5ec48628b200d9f9c5c8689 is 50, key is test_row_0/C:col10/1731187105943/Put/seqid=0 2024-11-09T21:18:30,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742461_1637 (size=12983) 2024-11-09T21:18:30,147 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/ce1e1e1df5ec48628b200d9f9c5c8689 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ce1e1e1df5ec48628b200d9f9c5c8689 2024-11-09T21:18:30,150 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 172f8346f592a41a81f031b40097d85b/C of 172f8346f592a41a81f031b40097d85b into ce1e1e1df5ec48628b200d9f9c5c8689(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:30,150 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:30,150 INFO [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/C, priority=12, startTime=1731187109642; duration=0sec 2024-11-09T21:18:30,151 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:30,151 DEBUG [RS:0;9514d6d5e5be:38413-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:C 2024-11-09T21:18:30,526 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/4d46d0fb906d4a8b99071718ce876227 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4d46d0fb906d4a8b99071718ce876227 2024-11-09T21:18:30,529 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 172f8346f592a41a81f031b40097d85b/A of 172f8346f592a41a81f031b40097d85b into 4d46d0fb906d4a8b99071718ce876227(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-09T21:18:30,529 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 172f8346f592a41a81f031b40097d85b: 2024-11-09T21:18:30,529 INFO [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., storeName=172f8346f592a41a81f031b40097d85b/A, priority=12, startTime=1731187109642; duration=0sec 2024-11-09T21:18:30,529 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-09T21:18:30,529 DEBUG [RS:0;9514d6d5e5be:38413-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 172f8346f592a41a81f031b40097d85b:A 2024-11-09T21:18:33,224 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-09T21:18:33,987 INFO [Thread-2458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2458. 2024-11-09T21:18:33,987 DEBUG [Thread-2458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-09T21:18:33,987 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:33,987 DEBUG [Thread-2458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:33,987 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 51 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3220 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3007 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3029 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3227 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3002 2024-11-09T21:18:33,988 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-09T21:18:33,988 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:18:33,988 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:18:33,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:33,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:33,988 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:33,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-09T21:18:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-09T21:18:33,991 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187113991"}]},"ts":"1731187113991"} 2024-11-09T21:18:33,992 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-09T21:18:33,992 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-09T21:18:33,992 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-09T21:18:33,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, UNASSIGN}] 2024-11-09T21:18:33,994 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, UNASSIGN 2024-11-09T21:18:33,994 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=CLOSING, regionLocation=9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:33,995 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=174, ppid=173, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, UNASSIGN because future has completed 2024-11-09T21:18:33,995 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-09T21:18:33,995 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857}] 2024-11-09T21:18:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-09T21:18:34,147 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(122): Close 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:34,147 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-09T21:18:34,147 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1722): Closing 172f8346f592a41a81f031b40097d85b, disabling compactions & flushes 2024-11-09T21:18:34,147 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:34,147 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:34,147 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. after waiting 0 ms 2024-11-09T21:18:34,147 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:34,147 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(2902): Flushing 172f8346f592a41a81f031b40097d85b 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-09T21:18:34,148 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=A 2024-11-09T21:18:34,148 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:34,148 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=B 2024-11-09T21:18:34,148 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:34,148 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 172f8346f592a41a81f031b40097d85b, store=C 2024-11-09T21:18:34,148 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-09T21:18:34,152 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093f5a4ce459634d50b806068edba5e17e_172f8346f592a41a81f031b40097d85b is 50, key is test_row_0/A:col10/1731187108116/Put/seqid=0 2024-11-09T21:18:34,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742462_1638 (size=9914) 2024-11-09T21:18:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-09T21:18:34,555 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T21:18:34,558 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411093f5a4ce459634d50b806068edba5e17e_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093f5a4ce459634d50b806068edba5e17e_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:34,559 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/d25df21ad6c94917b26d1ef5e1e20d1c, store: [table=TestAcidGuarantees family=A region=172f8346f592a41a81f031b40097d85b] 2024-11-09T21:18:34,559 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/d25df21ad6c94917b26d1ef5e1e20d1c is 175, key is test_row_0/A:col10/1731187108116/Put/seqid=0 2024-11-09T21:18:34,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742463_1639 (size=22561) 2024-11-09T21:18:34,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-09T21:18:34,962 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=298, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/d25df21ad6c94917b26d1ef5e1e20d1c 2024-11-09T21:18:34,967 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/b9a4253fe9b342adb31e2c772e36f49b is 50, key is test_row_0/B:col10/1731187108116/Put/seqid=0 2024-11-09T21:18:34,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742464_1640 (size=9857) 2024-11-09T21:18:35,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-09T21:18:35,371 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/b9a4253fe9b342adb31e2c772e36f49b 2024-11-09T21:18:35,418 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/f8492bdc6bb643ae952978e189939b3a is 50, key is test_row_0/C:col10/1731187108116/Put/seqid=0 2024-11-09T21:18:35,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742465_1641 (size=9857) 2024-11-09T21:18:35,822 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=298 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/f8492bdc6bb643ae952978e189939b3a 2024-11-09T21:18:35,825 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/A/d25df21ad6c94917b26d1ef5e1e20d1c as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d25df21ad6c94917b26d1ef5e1e20d1c 2024-11-09T21:18:35,828 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d25df21ad6c94917b26d1ef5e1e20d1c, entries=100, sequenceid=298, filesize=22.0 K 2024-11-09T21:18:35,828 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/B/b9a4253fe9b342adb31e2c772e36f49b as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/b9a4253fe9b342adb31e2c772e36f49b 2024-11-09T21:18:35,830 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/b9a4253fe9b342adb31e2c772e36f49b, entries=100, sequenceid=298, filesize=9.6 K 2024-11-09T21:18:35,831 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/.tmp/C/f8492bdc6bb643ae952978e189939b3a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f8492bdc6bb643ae952978e189939b3a 2024-11-09T21:18:35,833 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f8492bdc6bb643ae952978e189939b3a, entries=100, sequenceid=298, filesize=9.6 K 2024-11-09T21:18:35,833 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 172f8346f592a41a81f031b40097d85b in 1686ms, sequenceid=298, compaction requested=false 2024-11-09T21:18:35,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/a606fd6909944da6ab6dcf683372a93e, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d84ec7db34754ccab2195a5936f3cbfa, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4aea51f6db6446ec9fd7d64d8bc17432, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/b9642ffa23fe40d19bed71f38e8b55e0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/8d9e0321696247f7be1d82574b007172, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3] to archive 2024-11-09T21:18:35,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:18:35,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/418dd56e4a5046d2b2836632e0958e96 2024-11-09T21:18:35,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/583d7f6eb615429180228ca8ce643401 2024-11-09T21:18:35,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/9136ee24198a46ec8155683f7b56298b 2024-11-09T21:18:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/a606fd6909944da6ab6dcf683372a93e to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/a606fd6909944da6ab6dcf683372a93e 2024-11-09T21:18:35,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6765f5a650b8472eac4c5362ef8224dc 2024-11-09T21:18:35,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e2e40e4d4a034a8eba439ff252506dfc 2024-11-09T21:18:35,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d84ec7db34754ccab2195a5936f3cbfa to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d84ec7db34754ccab2195a5936f3cbfa 2024-11-09T21:18:35,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/dfba6c4e565949aabe1327e7db1f8d35 2024-11-09T21:18:35,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/04f7cdbf23af4015875c63205f9e1ad0 2024-11-09T21:18:35,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4aea51f6db6446ec9fd7d64d8bc17432 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4aea51f6db6446ec9fd7d64d8bc17432 2024-11-09T21:18:35,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/7a22593703e44c1191d5ac2d3399d682 2024-11-09T21:18:35,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4181cd058f1b44eb932a2c1b718d4fd7 2024-11-09T21:18:35,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/fd874d505cef483b9dbc711a226089e4 2024-11-09T21:18:35,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/b9642ffa23fe40d19bed71f38e8b55e0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/b9642ffa23fe40d19bed71f38e8b55e0 2024-11-09T21:18:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e585e8a8266c42a0940bf83226080b21 2024-11-09T21:18:35,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/07e445c3aa164b6b835ddba707cfebd4 2024-11-09T21:18:35,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/8d9e0321696247f7be1d82574b007172 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/8d9e0321696247f7be1d82574b007172 2024-11-09T21:18:35,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/ff1a7f4a387044148d1b61258104e011 2024-11-09T21:18:35,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/6825c6841d4d4136b3b85ed83ea32f5d 2024-11-09T21:18:35,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/e1dba02605f14b1e91fc7fd1550306b3 2024-11-09T21:18:35,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/fd7b6bbb2cfa461ba8693fd0a6c89e84, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/59561a936c1242d5ad036b4ad4c479c0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/00d5457b990545558323668da39f9035, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2a627725546441389897140d27f8673f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/13177f786ee6435f9771579cef802950, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/6f475706008948789ea1c931d51fd83f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/5b81833f6365449ba021804fb093a963, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/15b774a6c5734b98a77e9c19cfac0b37, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2e22423180da4dfa8970c7161896dc05, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/38d41d9a457e43289cb7d199dd389d80, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/64e1cffec98a4698bcee81f8314813b0, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7d28d84f53444e18a2cbb16a3361dde6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ebeb208e3c31458aa9672228df8e5c14, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/4ae9efc50140407f808a8f1538af2ffe, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/9894ee1ed5a54e0d896f6795a70b3c67, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/35378fc3122746adbdbf4a3e814c3bf4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/096255bd859c4a71b03264195d5ff50f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ad6e58f1b4ac439a8708fedd40dad45f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/e88771761cd44804b311bf3d481925ef, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/f12eadff7211465c815bc0873dbecbde] to archive 2024-11-09T21:18:35,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:18:35,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/fd7b6bbb2cfa461ba8693fd0a6c89e84 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/fd7b6bbb2cfa461ba8693fd0a6c89e84 2024-11-09T21:18:35,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/59561a936c1242d5ad036b4ad4c479c0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/59561a936c1242d5ad036b4ad4c479c0 2024-11-09T21:18:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/00d5457b990545558323668da39f9035 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/00d5457b990545558323668da39f9035 2024-11-09T21:18:35,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2a627725546441389897140d27f8673f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2a627725546441389897140d27f8673f 2024-11-09T21:18:35,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/13177f786ee6435f9771579cef802950 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/13177f786ee6435f9771579cef802950 2024-11-09T21:18:35,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/6f475706008948789ea1c931d51fd83f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/6f475706008948789ea1c931d51fd83f 2024-11-09T21:18:35,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/5b81833f6365449ba021804fb093a963 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/5b81833f6365449ba021804fb093a963 2024-11-09T21:18:35,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/15b774a6c5734b98a77e9c19cfac0b37 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/15b774a6c5734b98a77e9c19cfac0b37 2024-11-09T21:18:35,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2e22423180da4dfa8970c7161896dc05 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/2e22423180da4dfa8970c7161896dc05 2024-11-09T21:18:35,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/38d41d9a457e43289cb7d199dd389d80 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/38d41d9a457e43289cb7d199dd389d80 2024-11-09T21:18:35,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/64e1cffec98a4698bcee81f8314813b0 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/64e1cffec98a4698bcee81f8314813b0 2024-11-09T21:18:35,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7d28d84f53444e18a2cbb16a3361dde6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7d28d84f53444e18a2cbb16a3361dde6 2024-11-09T21:18:35,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ebeb208e3c31458aa9672228df8e5c14 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ebeb208e3c31458aa9672228df8e5c14 2024-11-09T21:18:35,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/4ae9efc50140407f808a8f1538af2ffe to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/4ae9efc50140407f808a8f1538af2ffe 2024-11-09T21:18:35,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/9894ee1ed5a54e0d896f6795a70b3c67 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/9894ee1ed5a54e0d896f6795a70b3c67 2024-11-09T21:18:35,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/35378fc3122746adbdbf4a3e814c3bf4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/35378fc3122746adbdbf4a3e814c3bf4 2024-11-09T21:18:35,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/096255bd859c4a71b03264195d5ff50f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/096255bd859c4a71b03264195d5ff50f 2024-11-09T21:18:35,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ad6e58f1b4ac439a8708fedd40dad45f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/ad6e58f1b4ac439a8708fedd40dad45f 2024-11-09T21:18:35,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/e88771761cd44804b311bf3d481925ef to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/e88771761cd44804b311bf3d481925ef 2024-11-09T21:18:35,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/f12eadff7211465c815bc0873dbecbde to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/f12eadff7211465c815bc0873dbecbde 2024-11-09T21:18:35,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8b115dd66441482582791ac33e967ba2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/2f3f517cb6eb41f688998300dd115fa2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/a6c4eaffa86748758d6ab23b4f1976ee, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ba5059e8e3dc457799bcf9f5954fb056, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f2eb563dfdca46fcb25aaa923067e35d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/1611186e516040f99aba56113ebf92b4, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/10e0e19889164b828a320918ed98f12b, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/af787d025fbd4a049cef3f4baa7febdb, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/73fbf675da4f4cb5b291fdb02c8c65b1, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/30ce283e1c2547ac8e814c4156cbbe23, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/458ae283c4ca4eee8ebc1d8dcbb0edf2, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ded864afcbfd4f10a21e41045354c90a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/caca1021a6044c11aa51b7bf20dfa4a6, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/5d02974c946745d7986d1424d47f656a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8bd0aab4badc450faa769eff1b9c125d, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/3001b776a23945f38058021d70657d6f, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/84b591699a7d47aa92af19fc830a2acf, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/bbc6ce9602884934b5f04ce87fb10e52, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/e1b7587a6f834557bdcebe1627290c3a, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/fa7d72b002ca480182f044a3977eb08c] to archive 2024-11-09T21:18:35,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-09T21:18:35,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8b115dd66441482582791ac33e967ba2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8b115dd66441482582791ac33e967ba2 2024-11-09T21:18:35,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/2f3f517cb6eb41f688998300dd115fa2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/2f3f517cb6eb41f688998300dd115fa2 2024-11-09T21:18:35,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/a6c4eaffa86748758d6ab23b4f1976ee to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/a6c4eaffa86748758d6ab23b4f1976ee 2024-11-09T21:18:35,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ba5059e8e3dc457799bcf9f5954fb056 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ba5059e8e3dc457799bcf9f5954fb056 2024-11-09T21:18:35,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f2eb563dfdca46fcb25aaa923067e35d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f2eb563dfdca46fcb25aaa923067e35d 2024-11-09T21:18:35,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/1611186e516040f99aba56113ebf92b4 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/1611186e516040f99aba56113ebf92b4 2024-11-09T21:18:35,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/10e0e19889164b828a320918ed98f12b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/10e0e19889164b828a320918ed98f12b 2024-11-09T21:18:35,868 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/af787d025fbd4a049cef3f4baa7febdb to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/af787d025fbd4a049cef3f4baa7febdb 2024-11-09T21:18:35,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/73fbf675da4f4cb5b291fdb02c8c65b1 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/73fbf675da4f4cb5b291fdb02c8c65b1 2024-11-09T21:18:35,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/30ce283e1c2547ac8e814c4156cbbe23 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/30ce283e1c2547ac8e814c4156cbbe23 2024-11-09T21:18:35,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/458ae283c4ca4eee8ebc1d8dcbb0edf2 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/458ae283c4ca4eee8ebc1d8dcbb0edf2 2024-11-09T21:18:35,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ded864afcbfd4f10a21e41045354c90a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ded864afcbfd4f10a21e41045354c90a 2024-11-09T21:18:35,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/caca1021a6044c11aa51b7bf20dfa4a6 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/caca1021a6044c11aa51b7bf20dfa4a6 2024-11-09T21:18:35,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/5d02974c946745d7986d1424d47f656a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/5d02974c946745d7986d1424d47f656a 2024-11-09T21:18:35,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8bd0aab4badc450faa769eff1b9c125d to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/8bd0aab4badc450faa769eff1b9c125d 2024-11-09T21:18:35,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/3001b776a23945f38058021d70657d6f to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/3001b776a23945f38058021d70657d6f 2024-11-09T21:18:35,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/84b591699a7d47aa92af19fc830a2acf to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/84b591699a7d47aa92af19fc830a2acf 2024-11-09T21:18:35,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/bbc6ce9602884934b5f04ce87fb10e52 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/bbc6ce9602884934b5f04ce87fb10e52 2024-11-09T21:18:35,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/e1b7587a6f834557bdcebe1627290c3a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/e1b7587a6f834557bdcebe1627290c3a 2024-11-09T21:18:35,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/fa7d72b002ca480182f044a3977eb08c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/fa7d72b002ca480182f044a3977eb08c 2024-11-09T21:18:35,878 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/recovered.edits/301.seqid, newMaxSeqId=301, maxSeqId=4 2024-11-09T21:18:35,879 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. 2024-11-09T21:18:35,879 DEBUG [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] regionserver.HRegion(1676): Region close journal for 172f8346f592a41a81f031b40097d85b: Waiting for close lock at 1731187114147Running coprocessor pre-close hooks at 1731187114147Disabling compacts and flushes for region at 1731187114147Disabling writes for close at 1731187114147Obtaining lock to block concurrent updates at 1731187114147Preparing flush snapshotting stores in 172f8346f592a41a81f031b40097d85b at 1731187114147Finished memstore snapshotting TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731187114148 (+1 ms)Flushing stores of TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b. at 1731187114148Flushing 172f8346f592a41a81f031b40097d85b/A: creating writer at 1731187114148Flushing mob file 172f8346f592a41a81f031b40097d85b/A: appending metadata at 1731187114152 (+4 ms)Flushing mob file 172f8346f592a41a81f031b40097d85b/A: closing flushed file at 1731187114152Flushing 172f8346f592a41a81f031b40097d85b/A: appending metadata at 1731187114559 (+407 ms)Flushing 172f8346f592a41a81f031b40097d85b/A: closing flushed file at 1731187114559Flushing 172f8346f592a41a81f031b40097d85b/B: creating writer at 1731187114965 (+406 ms)Flushing 172f8346f592a41a81f031b40097d85b/B: appending metadata at 1731187114967 (+2 ms)Flushing 172f8346f592a41a81f031b40097d85b/B: closing flushed file at 1731187114967Flushing 172f8346f592a41a81f031b40097d85b/C: creating writer at 1731187115416 (+449 ms)Flushing 172f8346f592a41a81f031b40097d85b/C: appending metadata at 1731187115418 (+2 ms)Flushing 172f8346f592a41a81f031b40097d85b/C: closing flushed file at 1731187115418Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba96b80: reopening flushed file at 1731187115824 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@fcaa77e: reopening flushed file at 1731187115828 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ed552f4: reopening flushed file at 1731187115831 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 172f8346f592a41a81f031b40097d85b in 1686ms, sequenceid=298, compaction requested=false at 1731187115833 (+2 ms)Writing region close event to WAL at 1731187115876 (+43 ms)Running coprocessor post-close hooks at 1731187115879 (+3 ms)Closed at 1731187115879 2024-11-09T21:18:35,880 INFO [RS_CLOSE_REGION-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_REGION, pid=175}] handler.UnassignRegionHandler(157): Closed 172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:35,880 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=174 updating hbase:meta row=172f8346f592a41a81f031b40097d85b, regionState=CLOSED 2024-11-09T21:18:35,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=175, ppid=174, state=RUNNABLE, hasLock=false; CloseRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 because future has completed 2024-11-09T21:18:35,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-09T21:18:35,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; CloseRegionProcedure 172f8346f592a41a81f031b40097d85b, server=9514d6d5e5be,38413,1731186934857 in 1.8870 sec 2024-11-09T21:18:35,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=174, resume processing ppid=173 2024-11-09T21:18:35,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, ppid=173, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=172f8346f592a41a81f031b40097d85b, UNASSIGN in 1.8900 sec 2024-11-09T21:18:35,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-09T21:18:35,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8930 sec 2024-11-09T21:18:35,887 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731187115886"}]},"ts":"1731187115886"} 2024-11-09T21:18:35,887 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-09T21:18:35,887 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-09T21:18:35,889 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8990 sec 2024-11-09T21:18:36,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-09T21:18:36,137 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:36,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-09T21:18:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:36,139 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=176, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:36,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-09T21:18:36,139 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=176, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:36,141 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,143 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C, FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/recovered.edits] 2024-11-09T21:18:36,144 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4d46d0fb906d4a8b99071718ce876227 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/4d46d0fb906d4a8b99071718ce876227 2024-11-09T21:18:36,145 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d25df21ad6c94917b26d1ef5e1e20d1c to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/A/d25df21ad6c94917b26d1ef5e1e20d1c 2024-11-09T21:18:36,147 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7901b4afe23c4955a2d8cd0c39dded4b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/7901b4afe23c4955a2d8cd0c39dded4b 2024-11-09T21:18:36,147 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/b9a4253fe9b342adb31e2c772e36f49b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/B/b9a4253fe9b342adb31e2c772e36f49b 2024-11-09T21:18:36,149 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ce1e1e1df5ec48628b200d9f9c5c8689 to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/ce1e1e1df5ec48628b200d9f9c5c8689 2024-11-09T21:18:36,150 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f8492bdc6bb643ae952978e189939b3a to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/C/f8492bdc6bb643ae952978e189939b3a 2024-11-09T21:18:36,152 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/recovered.edits/301.seqid to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b/recovered.edits/301.seqid 2024-11-09T21:18:36,152 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/default/TestAcidGuarantees/172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,152 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-09T21:18:36,152 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T21:18:36,153 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-09T21:18:36,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091b9bed6702ce48959a4e0091bf950e40_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411091b9bed6702ce48959a4e0091bf950e40_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,155 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093f5a4ce459634d50b806068edba5e17e_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411093f5a4ce459634d50b806068edba5e17e_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,156 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094badf7ca72ec41edbbc8f3399c55baa5_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411094badf7ca72ec41edbbc8f3399c55baa5_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,157 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109551af29681d444aeadff4834464fce0c_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109551af29681d444aeadff4834464fce0c_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,158 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095e9b23449b244d2c8e21cd502fe9c02e_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411095e9b23449b244d2c8e21cd502fe9c02e_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109689e1a5e6fb44f02aa9e5b66b5c66c15_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109689e1a5e6fb44f02aa9e5b66b5c66c15_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,159 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109689e6d13533e4826b96e397a48a8b5ab_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109689e6d13533e4826b96e397a48a8b5ab_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,160 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099121ca2daea54da7a9c6ff93a5eea085_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411099121ca2daea54da7a9c6ff93a5eea085_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,161 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a30aa7f91fbe4ed28682ddef08bd2255_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a30aa7f91fbe4ed28682ddef08bd2255_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a725183e2b9e4aa09ec38ea700bc830c_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109a725183e2b9e4aa09ec38ea700bc830c_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,162 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b3a095165af0460b82e0c5272f495c59_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b3a095165af0460b82e0c5272f495c59_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,163 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b740f03531d84fcd8db1fc37371f95e3_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109b740f03531d84fcd8db1fc37371f95e3_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c76521819db348c3996bbdb332ced7c8_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109c76521819db348c3996bbdb332ced7c8_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,164 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dafd10bc8656484e9758a87e319cf36f_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109dafd10bc8656484e9758a87e319cf36f_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,165 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109decca35739324cdba44f7332d96e32ef_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109decca35739324cdba44f7332d96e32ef_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,166 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e4049cdbf7204218a0bcd422c2ff78a8_172f8346f592a41a81f031b40097d85b to hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241109e4049cdbf7204218a0bcd422c2ff78a8_172f8346f592a41a81f031b40097d85b 2024-11-09T21:18:36,166 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-09T21:18:36,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=176, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:36,169 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-09T21:18:36,171 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-09T21:18:36,171 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=176, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:36,171 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-09T21:18:36,172 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731187116171"}]},"ts":"9223372036854775807"} 2024-11-09T21:18:36,173 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-09T21:18:36,173 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 172f8346f592a41a81f031b40097d85b, NAME => 'TestAcidGuarantees,,1731187083945.172f8346f592a41a81f031b40097d85b.', STARTKEY => '', ENDKEY => ''}] 2024-11-09T21:18:36,173 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-09T21:18:36,173 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731187116173"}]},"ts":"9223372036854775807"} 2024-11-09T21:18:36,174 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-09T21:18:36,175 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=176, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-09T21:18:36,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 38 msec 2024-11-09T21:18:36,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=34371 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-09T21:18:36,247 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-09T21:18:36,249 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-09T21:18:36,255 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 251), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=458 (was 494), ProcessCount=11 (was 11), AvailableMemoryMB=2962 (was 3149) 2024-11-09T21:18:36,255 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T21:18:36,255 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:18:36,255 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:18:36,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:36,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:36,255 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T21:18:36,255 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T21:18:36,255 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=841754823, stopped=false 2024-11-09T21:18:36,256 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9514d6d5e5be,34371,1731186934105 2024-11-09T21:18:36,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:18:36,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T21:18:36,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:18:36,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:18:36,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:18:36,389 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T21:18:36,389 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:18:36,389 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:36,390 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9514d6d5e5be,38413,1731186934857' ***** 2024-11-09T21:18:36,390 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T21:18:36,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:18:36,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T21:18:36,390 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T21:18:36,390 INFO [RS:0;9514d6d5e5be:38413 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T21:18:36,391 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(959): stopping server 9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9514d6d5e5be:38413. 2024-11-09T21:18:36,391 DEBUG [RS:0;9514d6d5e5be:38413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T21:18:36,391 DEBUG [RS:0;9514d6d5e5be:38413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T21:18:36,391 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T21:18:36,392 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-09T21:18:36,392 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-09T21:18:36,392 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:36,392 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T21:18:36,392 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T21:18:36,392 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T21:18:36,392 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T21:18:36,392 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T21:18:36,392 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-09T21:18:36,414 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/info/b93dab7d30b048708a44c7d4311f096a is 95, key is TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df./info:/1731186964536/DeleteFamily/seqid=0 2024-11-09T21:18:36,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742466_1642 (size=6514) 2024-11-09T21:18:36,425 INFO [regionserver/9514d6d5e5be:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:18:36,592 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:36,792 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:36,818 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/info/b93dab7d30b048708a44c7d4311f096a 2024-11-09T21:18:36,829 INFO [regionserver/9514d6d5e5be:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T21:18:36,829 INFO [regionserver/9514d6d5e5be:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T21:18:36,834 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/ns/d8a1b6ea4bd942c79a767705effa0eff is 93, key is TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df./ns:/1731186964486/DeleteFamily/seqid=0 2024-11-09T21:18:36,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742467_1643 (size=5906) 2024-11-09T21:18:36,992 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:37,193 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:37,237 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/ns/d8a1b6ea4bd942c79a767705effa0eff 2024-11-09T21:18:37,252 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/rep_barrier/6c05db3e394942c7a933579c56359c8f is 102, key is TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df./rep_barrier:/1731186964486/DeleteFamily/seqid=0 2024-11-09T21:18:37,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742468_1644 (size=6025) 2024-11-09T21:18:37,393 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:37,593 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:37,655 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/rep_barrier/6c05db3e394942c7a933579c56359c8f 2024-11-09T21:18:37,671 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/table/fe70f39fe8454ab3b22d14a0f81e7f5e is 96, key is TestAcidGuarantees,,1731186939471.ac01e9e325a29ffdc19c65c95fe0a1df./table:/1731186964486/DeleteFamily/seqid=0 2024-11-09T21:18:37,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742469_1645 (size=5936) 2024-11-09T21:18:37,793 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:37,912 INFO [regionserver/9514d6d5e5be:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: ExecutorStatusChore was stopped 2024-11-09T21:18:37,993 DEBUG [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-09T21:18:38,074 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/table/fe70f39fe8454ab3b22d14a0f81e7f5e 2024-11-09T21:18:38,078 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/info/b93dab7d30b048708a44c7d4311f096a as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/info/b93dab7d30b048708a44c7d4311f096a 2024-11-09T21:18:38,081 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/info/b93dab7d30b048708a44c7d4311f096a, entries=12, sequenceid=90, filesize=6.4 K 2024-11-09T21:18:38,081 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/ns/d8a1b6ea4bd942c79a767705effa0eff as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/ns/d8a1b6ea4bd942c79a767705effa0eff 2024-11-09T21:18:38,084 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/ns/d8a1b6ea4bd942c79a767705effa0eff, entries=8, sequenceid=90, filesize=5.8 K 2024-11-09T21:18:38,084 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/rep_barrier/6c05db3e394942c7a933579c56359c8f as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/rep_barrier/6c05db3e394942c7a933579c56359c8f 2024-11-09T21:18:38,087 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/rep_barrier/6c05db3e394942c7a933579c56359c8f, entries=6, sequenceid=90, filesize=5.9 K 2024-11-09T21:18:38,087 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/.tmp/table/fe70f39fe8454ab3b22d14a0f81e7f5e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/table/fe70f39fe8454ab3b22d14a0f81e7f5e 2024-11-09T21:18:38,090 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/table/fe70f39fe8454ab3b22d14a0f81e7f5e, entries=7, sequenceid=90, filesize=5.8 K 2024-11-09T21:18:38,091 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1699ms, sequenceid=90, compaction requested=false 2024-11-09T21:18:38,094 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-09T21:18:38,094 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T21:18:38,095 INFO [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T21:18:38,095 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731187116392Running coprocessor pre-close hooks at 1731187116392Disabling compacts and flushes for region at 1731187116392Disabling writes for close at 1731187116392Obtaining lock to block concurrent updates at 1731187116392Preparing flush snapshotting stores in 1588230740 at 1731187116392Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731187116393 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731187116393Flushing 1588230740/info: creating writer at 1731187116393Flushing 1588230740/info: appending metadata at 1731187116413 (+20 ms)Flushing 1588230740/info: closing flushed file at 1731187116413Flushing 1588230740/ns: creating writer at 1731187116821 (+408 ms)Flushing 1588230740/ns: appending metadata at 1731187116834 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731187116834Flushing 1588230740/rep_barrier: creating writer at 1731187117240 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731187117252 (+12 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731187117252Flushing 1588230740/table: creating writer at 1731187117658 (+406 ms)Flushing 1588230740/table: appending metadata at 1731187117671 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731187117671Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@340c019a: reopening flushed file at 1731187118077 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@608660f0: reopening flushed file at 1731187118081 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c0d9b25: reopening flushed file at 1731187118084 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@28424d25: reopening flushed file at 1731187118087 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1699ms, sequenceid=90, compaction requested=false at 1731187118091 (+4 ms)Writing region close event to WAL at 1731187118092 (+1 ms)Running coprocessor post-close hooks at 1731187118094 (+2 ms)Closed at 1731187118094 2024-11-09T21:18:38,095 DEBUG [RS_CLOSE_META-regionserver/9514d6d5e5be:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T21:18:38,194 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(976): stopping server 9514d6d5e5be,38413,1731186934857; all regions closed. 2024-11-09T21:18:38,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741834_1010 (size=25344) 2024-11-09T21:18:38,199 DEBUG [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/oldWALs 2024-11-09T21:18:38,199 INFO [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9514d6d5e5be%2C38413%2C1731186934857.meta:.meta(num 1731186939018) 2024-11-09T21:18:38,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741832_1008 (size=19730537) 2024-11-09T21:18:38,202 DEBUG [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/oldWALs 2024-11-09T21:18:38,202 INFO [RS:0;9514d6d5e5be:38413 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9514d6d5e5be%2C38413%2C1731186934857:(num 1731186937977) 2024-11-09T21:18:38,202 DEBUG [RS:0;9514d6d5e5be:38413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T21:18:38,202 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T21:18:38,202 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:18:38,203 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.ChoreService(370): Chore service for: regionserver/9514d6d5e5be:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-09T21:18:38,203 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:18:38,203 INFO [regionserver/9514d6d5e5be:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:18:38,204 INFO [RS:0;9514d6d5e5be:38413 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38413 2024-11-09T21:18:38,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T21:18:38,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9514d6d5e5be,38413,1731186934857 2024-11-09T21:18:38,273 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:18:38,274 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9514d6d5e5be,38413,1731186934857] 2024-11-09T21:18:38,294 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9514d6d5e5be,38413,1731186934857 already deleted, retry=false 2024-11-09T21:18:38,294 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9514d6d5e5be,38413,1731186934857 expired; onlineServers=0 2024-11-09T21:18:38,294 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9514d6d5e5be,34371,1731186934105' ***** 2024-11-09T21:18:38,294 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T21:18:38,294 INFO [M:0;9514d6d5e5be:34371 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T21:18:38,295 INFO [M:0;9514d6d5e5be:34371 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T21:18:38,295 DEBUG [M:0;9514d6d5e5be:34371 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T21:18:38,295 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T21:18:38,295 DEBUG [M:0;9514d6d5e5be:34371 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T21:18:38,295 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.large.0-1731186937661 {}] cleaner.HFileCleaner(306): Exit Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.large.0-1731186937661,5,FailOnTimeoutGroup] 2024-11-09T21:18:38,295 DEBUG [master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.small.0-1731186937661 {}] cleaner.HFileCleaner(306): Exit Thread[master/9514d6d5e5be:0:becomeActiveMaster-HFileCleaner.small.0-1731186937661,5,FailOnTimeoutGroup] 2024-11-09T21:18:38,295 INFO [M:0;9514d6d5e5be:34371 {}] hbase.ChoreService(370): Chore service for: master/9514d6d5e5be:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T21:18:38,295 INFO [M:0;9514d6d5e5be:34371 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T21:18:38,295 DEBUG [M:0;9514d6d5e5be:34371 {}] master.HMaster(1795): Stopping service threads 2024-11-09T21:18:38,295 INFO [M:0;9514d6d5e5be:34371 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T21:18:38,295 INFO [M:0;9514d6d5e5be:34371 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T21:18:38,296 ERROR [M:0;9514d6d5e5be:34371 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-09T21:18:38,296 INFO [M:0;9514d6d5e5be:34371 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T21:18:38,296 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T21:18:38,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T21:18:38,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T21:18:38,305 DEBUG [M:0;9514d6d5e5be:34371 {}] zookeeper.ZKUtil(347): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T21:18:38,305 WARN [M:0;9514d6d5e5be:34371 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T21:18:38,305 INFO [M:0;9514d6d5e5be:34371 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/.lastflushedseqids 2024-11-09T21:18:38,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742470_1646 (size=72) 2024-11-09T21:18:38,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:18:38,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38413-0x101217439d90001, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:18:38,384 INFO [RS:0;9514d6d5e5be:38413 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:18:38,384 INFO [RS:0;9514d6d5e5be:38413 {}] regionserver.HRegionServer(1031): Exiting; stopping=9514d6d5e5be,38413,1731186934857; zookeeper connection closed. 2024-11-09T21:18:38,385 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@417c8d78 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@417c8d78 2024-11-09T21:18:38,385 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-09T21:18:38,713 INFO [M:0;9514d6d5e5be:34371 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T21:18:38,714 INFO [M:0;9514d6d5e5be:34371 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T21:18:38,714 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T21:18:38,714 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:18:38,714 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:18:38,714 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T21:18:38,714 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:18:38,714 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=777.18 KB heapSize=951.79 KB 2024-11-09T21:18:38,727 DEBUG [M:0;9514d6d5e5be:34371 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09a919d9186c4f1d8079fb830102257e is 82, key is hbase:meta,,1/info:regioninfo/1731186939099/Put/seqid=0 2024-11-09T21:18:38,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742471_1647 (size=5672) 2024-11-09T21:18:39,131 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2153 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09a919d9186c4f1d8079fb830102257e 2024-11-09T21:18:39,151 DEBUG [M:0;9514d6d5e5be:34371 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc484b707a4742d680f184d5b758eae0 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x9D/proc:d/1731187087008/Put/seqid=0 2024-11-09T21:18:39,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742472_1648 (size=46398) 2024-11-09T21:18:39,555 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=776.63 KB at sequenceid=2153 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc484b707a4742d680f184d5b758eae0 2024-11-09T21:18:39,558 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fc484b707a4742d680f184d5b758eae0 2024-11-09T21:18:39,571 DEBUG [M:0;9514d6d5e5be:34371 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b94dc20dcfc040c7b325add41fb6a1a5 is 69, key is 9514d6d5e5be,38413,1731186934857/rs:state/1731186937737/Put/seqid=0 2024-11-09T21:18:39,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073742473_1649 (size=5156) 2024-11-09T21:18:39,974 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2153 (bloomFilter=true), to=hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b94dc20dcfc040c7b325add41fb6a1a5 2024-11-09T21:18:39,978 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/09a919d9186c4f1d8079fb830102257e as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09a919d9186c4f1d8079fb830102257e 2024-11-09T21:18:39,981 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/09a919d9186c4f1d8079fb830102257e, entries=8, sequenceid=2153, filesize=5.5 K 2024-11-09T21:18:39,982 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fc484b707a4742d680f184d5b758eae0 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc484b707a4742d680f184d5b758eae0 2024-11-09T21:18:39,984 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fc484b707a4742d680f184d5b758eae0 2024-11-09T21:18:39,984 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fc484b707a4742d680f184d5b758eae0, entries=176, sequenceid=2153, filesize=45.3 K 2024-11-09T21:18:39,985 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b94dc20dcfc040c7b325add41fb6a1a5 as hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b94dc20dcfc040c7b325add41fb6a1a5 2024-11-09T21:18:39,988 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42817/user/jenkins/test-data/0e842e06-f124-7ff5-be0c-1cd0f6d26982/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b94dc20dcfc040c7b325add41fb6a1a5, entries=1, sequenceid=2153, filesize=5.0 K 2024-11-09T21:18:39,989 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(3140): Finished flush of dataSize ~777.18 KB/795833, heapSize ~951.49 KB/974328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2153, compaction requested=false 2024-11-09T21:18:39,991 INFO [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T21:18:39,991 DEBUG [M:0;9514d6d5e5be:34371 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731187118714Disabling compacts and flushes for region at 1731187118714Disabling writes for close at 1731187118714Obtaining lock to block concurrent updates at 1731187118714Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731187118714Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=795833, getHeapSize=974568, getOffHeapSize=0, getCellsCount=2354 at 1731187118714Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731187118715 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731187118715Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731187118727 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731187118727Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731187119134 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731187119151 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731187119151Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731187119558 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731187119570 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731187119570Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3681f845: reopening flushed file at 1731187119977 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7416fcd5: reopening flushed file at 1731187119981 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54ba4d8a: reopening flushed file at 1731187119985 (+4 ms)Finished flush of dataSize ~777.18 KB/795833, heapSize ~951.49 KB/974328, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1275ms, sequenceid=2153, compaction requested=false at 1731187119989 (+4 ms)Writing region close event to WAL at 1731187119990 (+1 ms)Closed at 1731187119990 2024-11-09T21:18:39,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43329 is added to blk_1073741830_1006 (size=937001) 2024-11-09T21:18:39,994 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T21:18:39,994 INFO [M:0;9514d6d5e5be:34371 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T21:18:39,994 INFO [M:0;9514d6d5e5be:34371 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:34371 2024-11-09T21:18:39,994 INFO [M:0;9514d6d5e5be:34371 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T21:18:40,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:18:40,141 INFO [M:0;9514d6d5e5be:34371 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T21:18:40,141 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:34371-0x101217439d90000, quorum=127.0.0.1:59129, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T21:18:40,146 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@292d50c4{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T21:18:40,147 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@667c6eba{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:18:40,148 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:18:40,148 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@72782cb0{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:18:40,148 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@54a7b62e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/hadoop.log.dir/,STOPPED} 2024-11-09T21:18:40,150 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T21:18:40,150 WARN [BP-1250636572-172.17.0.2-1731186930089 heartbeating to localhost/127.0.0.1:42817 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T21:18:40,150 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T21:18:40,150 WARN [BP-1250636572-172.17.0.2-1731186930089 heartbeating to localhost/127.0.0.1:42817 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1250636572-172.17.0.2-1731186930089 (Datanode Uuid f48c03a0-7fd7-4b12-8a65-5a278ca37ae8) service to localhost/127.0.0.1:42817 2024-11-09T21:18:40,153 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/data/data1/current/BP-1250636572-172.17.0.2-1731186930089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:18:40,153 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/cluster_2070bea1-1521-e1fd-fafd-989d45db8446/data/data2/current/BP-1250636572-172.17.0.2-1731186930089 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T21:18:40,153 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T21:18:40,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2341a969{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T21:18:40,160 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2d064751{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T21:18:40,160 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T21:18:40,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@130d168e{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T21:18:40,160 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1a58fd8a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3@2/hbase-server/target/test-data/6a0b8a6f-4e66-da67-8522-e6fc91ade775/hadoop.log.dir/,STOPPED} 2024-11-09T21:18:40,174 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T21:18:40,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down